==================================================================================================== ========================================= Pod describe ========================================= ==================================================================================================== Name: lodemon-64cbdcc7b4-vt4kk Namespace: xlou Priority: 0 Service Account: k8s-svc-acct-lodemon Node: gke-xlou-cdm-ds-0f601751-mc5d/10.142.0.43 Start Time: Sat, 13 Apr 2024 23:08:53 +0000 Labels: app=lodemon app.kubernetes.io/name=lodemon pod-template-hash=64cbdcc7b4 skaffold.dev/run-id=556a16b8-81bd-4d59-b918-4dea9653a150 Annotations: Status: Running IP: 10.106.46.134 IPs: IP: 10.106.46.134 Controlled By: ReplicaSet/lodemon-64cbdcc7b4 Containers: lodemon: Container ID: containerd://db3c72aed0a75916fcc8d1c8a6d91d42014eb5cbe12152fd2872db724ce93def Image: gcr.io/engineeringpit/lodestar-images/lodestarbox:master-stable Image ID: gcr.io/engineeringpit/lodestar-images/lodestarbox@sha256:06c53901427b11990b989f0a16f422c2c89c055d3393605a0d2d532652d23afc Port: 8080/TCP Host Port: 0/TCP Command: python3 Args: /lodestar/scripts/lodemon_run.py -W default State: Running Started: Sat, 13 Apr 2024 23:09:35 +0000 Ready: True Restart Count: 0 Limits: cpu: 2 memory: 2Gi Requests: cpu: 1 memory: 1Gi Liveness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Readiness: exec [cat /tmp/lodemon_alive] delay=20s timeout=1s period=10s #success=1 #failure=3 Environment: SKAFFOLD_PROFILE: medium Mounts: /lodestar/config/config.yaml from config (rw,path="config.yaml") /lodestar/config/deployments_to_monitor.yaml from deployments-config (rw,path="deployments_to_monitor.yaml") /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-rdddx (ro) Conditions: Type Status Initialized True Ready True ContainersReady True PodScheduled True Volumes: config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-config Optional: false deployments-config: Type: ConfigMap (a volume populated by a ConfigMap) Name: lodemon-deployments Optional: false kube-api-access-rdddx: Type: Projected (a volume that contains injected data from multiple sources) TokenExpirationSeconds: 3607 ConfigMapName: kube-root-ca.crt ConfigMapOptional: DownwardAPI: true QoS Class: Burstable Node-Selectors: Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s node.kubernetes.io/unreachable:NoExecute op=Exists for 300s Events: ==================================================================================================== =========================================== Pod logs =========================================== ==================================================================================================== * Serving Flask app 'lodemon_run' * Debug mode: off WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. * Running on all addresses (0.0.0.0) * Running on http://127.0.0.1:8080 * Running on http://10.106.46.134:8080 Press CTRL+C to quit 00:10:40 INFO 00:10:40 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:40 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:40 INFO [loop_until]: OK (rc = 0) 00:10:40 DEBUG --- stdout --- 00:10:40 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:40 DEBUG --- stderr --- 00:10:40 DEBUG 00:10:40 INFO 00:10:40 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:40 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:40 INFO [loop_until]: OK (rc = 0) 00:10:40 DEBUG --- stdout --- 00:10:40 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:40 DEBUG --- stderr --- 00:10:40 DEBUG 00:10:40 INFO 00:10:40 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:40 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:40 INFO [loop_until]: OK (rc = 0) 00:10:40 DEBUG --- stdout --- 00:10:40 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:40 DEBUG --- stderr --- 00:10:40 DEBUG 00:10:40 INFO 00:10:40 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:40 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:40 INFO [loop_until]: OK (rc = 0) 00:10:40 DEBUG --- stdout --- 00:10:40 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:40 DEBUG --- stderr --- 00:10:40 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:41 INFO [loop_until]: OK (rc = 0) 00:10:41 DEBUG --- stdout --- 00:10:41 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:41 DEBUG --- stderr --- 00:10:41 DEBUG 00:10:41 INFO 00:10:41 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:41 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:42 INFO 00:10:42 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:42 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:42 INFO [loop_until]: OK (rc = 0) 00:10:42 DEBUG --- stdout --- 00:10:42 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:42 DEBUG --- stderr --- 00:10:42 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:43 INFO 00:10:43 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:43 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:43 INFO [loop_until]: OK (rc = 0) 00:10:43 DEBUG --- stdout --- 00:10:43 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:43 DEBUG --- stderr --- 00:10:43 DEBUG 00:10:44 INFO 00:10:44 INFO [loop_until]: kubectl get services -o=jsonpath='{.items[?(@.metadata.labels.app=="kube-prometheus-stack-prometheus")]}' --all-namespaces 00:10:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:44 INFO [loop_until]: OK (rc = 0) 00:10:44 DEBUG --- stdout --- 00:10:44 DEBUG {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{"cloud.google.com/neg":"{\"ingress\":true}","meta.helm.sh/release-name":"prometheus-operator","meta.helm.sh/release-namespace":"monitoring"},"creationTimestamp":"2023-05-27T02:35:19Z","labels":{"app":"kube-prometheus-stack-prometheus","app.kubernetes.io/instance":"prometheus-operator","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/part-of":"kube-prometheus-stack","app.kubernetes.io/version":"46.4.1","chart":"kube-prometheus-stack-46.4.1","heritage":"Helm","release":"prometheus-operator","self-monitor":"true"},"managedFields":[{"apiVersion":"v1","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:meta.helm.sh/release-name":{},"f:meta.helm.sh/release-namespace":{}},"f:labels":{".":{},"f:app":{},"f:app.kubernetes.io/instance":{},"f:app.kubernetes.io/managed-by":{},"f:app.kubernetes.io/part-of":{},"f:app.kubernetes.io/version":{},"f:chart":{},"f:heritage":{},"f:release":{},"f:self-monitor":{}}},"f:spec":{"f:internalTrafficPolicy":{},"f:ports":{".":{},"k:{\"port\":9090,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:selector":{},"f:sessionAffinity":{},"f:type":{}}},"manager":"helm","operation":"Update","time":"2023-05-27T02:35:19Z"}],"name":"prometheus-operator-kube-p-prometheus","namespace":"monitoring","resourceVersion":"7148","uid":"eb1f35e8-cd91-4a12-8fe3-2e8bdf841da5"},"spec":{"clusterIP":"10.106.49.67","clusterIPs":["10.106.49.67"],"internalTrafficPolicy":"Cluster","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","ports":[{"name":"http-web","port":9090,"protocol":"TCP","targetPort":9090}],"selector":{"app.kubernetes.io/name":"prometheus","prometheus":"prometheus-operator-kube-p-prometheus"},"sessionAffinity":"None","type":"ClusterIP"},"status":{"loadBalancer":{}}} 00:10:44 DEBUG --- stderr --- 00:10:44 DEBUG 00:10:44 INFO Initializing monitoring instance threads 00:10:44 DEBUG Monitoring instance thread list: [, , , , , , , , , , , , , , , , , , , , , , , , , , , , ] 00:10:44 INFO Starting instance threads 00:10:44 INFO 00:10:44 INFO Thread started 00:10:44 INFO [loop_until]: kubectl --namespace=xlou top node 00:10:44 INFO 00:10:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:44 INFO Thread started 00:10:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 INFO Thread started 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG ----- output ----- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 DEBUG --- status code --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG { 00:10:44 DEBUG --- status code --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 INFO Thread started 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 INFO Thread started 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG "data": { 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 DEBUG --- http response --- 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 DEBUG ----- output ----- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG --- status code --- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG { 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 DEBUG ----- output ----- 00:10:44 INFO Thread started 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG { 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG "result": [ 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG { 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 DEBUG { 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG { 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 DEBUG { 00:10:44 INFO Thread started 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG { 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG { 00:10:44 DEBUG "data": { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "status": "success", 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "data": { 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "data": { 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG { 00:10:44 DEBUG "data": { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "data": { 00:10:44 DEBUG "data": { 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG "data": { 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 DEBUG "data": { 00:10:44 INFO [loop_until]: OK (rc = 0) 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG "data": { 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG "data": { 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "data": { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "data": { 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG "resultType": "vector", 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049844" 00:10:44 INFO Thread started 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG --- stdout --- 00:10:44 DEBUG "result": [ 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG ----- output ----- 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "data": { 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "data": { 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG { 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "result": [ 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG "result": [ 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049844" 00:10:44 DEBUG "result": [ 00:10:44 INFO Thread started 00:10:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 5m 1134Mi am-7d597cf7d6-8vr8f 9m 1213Mi am-7d597cf7d6-9wz9h 11m 3329Mi ds-cts-0 8m 473Mi ds-cts-1 8m 478Mi ds-cts-2 8m 475Mi ds-idrepo-0 14m 4179Mi ds-idrepo-1 13m 4259Mi ds-idrepo-2 18m 4110Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 953Mi idm-b85f9d978-rl76f 4m 961Mi lodemon-64cbdcc7b4-vt4kk 1m 55Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 1m 1013Mi 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG }, 00:10:44 DEBUG "data": { 00:10:44 DEBUG "result": [ 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG "result": [ 00:10:44 DEBUG { 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "metric": { 00:10:44 DEBUG { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG --- stderr --- 00:10:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049844" 00:10:44 DEBUG "metric": { 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 ERROR [http_cmd]: ERROR 00:10:44 DEBUG "status": "success", 00:10:44 INFO Thread started 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG { 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "result": [ 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:10:44 DEBUG }, 00:10:44 DEBUG { 00:10:44 DEBUG }, 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG 00:10:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:10:44 DEBUG --- status code --- 00:10:44 ERROR --- status code --- 00:10:44 DEBUG "data": { 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "data": { 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:10:44 DEBUG { 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "metric": { 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "instance": "10.142.0.107:9100" 00:10:44 DEBUG "instance": "10.142.0.107:9100" 00:10:44 DEBUG }, 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:10:44 INFO All threads has been started 127.0.0.1 - - [14/Apr/2024 00:10:44] "GET /monitoring/start HTTP/1.1" 200 - 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:10:44 DEBUG { 00:10:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG }, 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "data": { 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 DEBUG "instance": "10.142.0.107:9100" 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:10:44 DEBUG 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG --- http response --- 00:10:44 ERROR --- http response --- 00:10:44 DEBUG { 00:10:44 DEBUG "result": [ 00:10:44 DEBUG 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG "metric": { 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG { 00:10:44 DEBUG "value": [ 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "result": [ 00:10:44 DEBUG }, 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG }, 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:10:44 DEBUG "metric": { 00:10:44 DEBUG { 00:10:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:10:44 DEBUG [...] 00:10:44 INFO [http_cmd]: http status code OK 00:10:44 DEBUG { 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG "metric": { 00:10:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:10:44 DEBUG 00:10:44 DEBUG --- status code --- 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "data": { 00:10:44 DEBUG 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG { 00:10:44 DEBUG }, 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG 00:10:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:10:44 DEBUG "data": { 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG --- http response --- 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "result": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "result": [ 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "data": { 00:10:44 DEBUG "data": { 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG "data": { 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:10:44 DEBUG 00:10:44 DEBUG { 00:10:44 DEBUG [...] 00:10:44 DEBUG [...] 00:10:44 DEBUG { 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG [...] 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG "pod": "ds-idrepo-2" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG ----- output ----- 00:10:44 DEBUG 00:10:44 DEBUG "metric": { 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "metric": { 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "result": [ 00:10:44 DEBUG "result": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "result": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG { 00:10:44 DEBUG 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG { 00:10:44 DEBUG { 00:10:44 DEBUG "pod": "ds-idrepo-2" 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:10:44 DEBUG { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG [...] 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "status": "success", 00:10:44 DEBUG [...] 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG "pod": "ds-idrepo-2" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "metric": { 00:10:44 DEBUG "metric": { 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "instance": "10.142.0.43:9100" 00:10:44 DEBUG "instance": "10.142.0.43:9100" 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 INFO [loop_until]: OK (rc = 0) 00:10:44 DEBUG 00:10:44 DEBUG "0" 00:10:44 DEBUG "data": { 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "pod": "ds-idrepo-2" 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG "pod": "ds-idrepo-0" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "instance": "10.142.0.43:9100" 00:10:44 DEBUG "pod": "ds-idrepo-0" 00:10:44 DEBUG "pod": "ds-idrepo-1" 00:10:44 DEBUG "instance": "10.142.0.107:9100" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "pod": "ds-idrepo-0" 00:10:44 DEBUG "0.011560702658705728" 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG [...] 00:10:44 DEBUG }, 00:10:44 DEBUG --- stdout --- 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG "resultType": "vector", 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG }, 00:10:44 DEBUG ] 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG "result": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "pod": "ds-idrepo-0" 00:10:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:10:44 DEBUG 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "5556.1584373304395" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG } 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "1160347648" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG [...] 00:10:44 DEBUG ] 00:10:44 DEBUG { 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 00:10:44 DEBUG "4444.926749864351" 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG }, 00:10:44 DEBUG }, 00:10:44 DEBUG "pod": "ds-idrepo-0" 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG ] 00:10:44 DEBUG "11329.966666666664" 00:10:44 DEBUG "6933.633333333333" 00:10:44 DEBUG ] 00:10:44 DEBUG 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 4919Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 66m 0% 2458Mi 2% gke-xlou-cdm-default-pool-1b0769c8-nhlf 53m 0% 3195Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 92m 0% 2968Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 64m 0% 6234Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 159m 0% 3947Mi 3% gke-xlou-cdm-default-pool-1b0769c8-xnmp 73m 0% 2887Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 62m 0% 2690Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 73m 0% 5447Mi 4% gke-xlou-cdm-ds-0f601751-j488 61m 0% 1810Mi 1% gke-xlou-cdm-ds-0f601751-mc5d 76m 0% 2205Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 91m 0% 4868Mi 4% gke-xlou-cdm-ds-0f601751-nmxm 69m 0% 5550Mi 4% gke-xlou-cdm-frontend-a8771548-i82n 63m 0% 4770Mi 8% 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG "metric": { 00:10:44 DEBUG }, 00:10:44 DEBUG [...] 00:10:44 DEBUG "0" 00:10:44 DEBUG "0" 00:10:44 DEBUG [...] 00:10:44 DEBUG ] 00:10:44 DEBUG "0" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG "0.0006666666666666673" 00:10:44 DEBUG "0" 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG "0" 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG }, 00:10:44 DEBUG --- stderr --- 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG "instance": "10.142.0.107:9100" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG }, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG }, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG "0" 00:10:44 DEBUG "0.033759777233650555" 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG [...] 00:10:44 DEBUG [...] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "pod": "ds-cts-1" 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "0" 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG "0" 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "0" 00:10:44 DEBUG }, 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG } 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG "0" 00:10:44 DEBUG ] 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG }, 00:10:44 DEBUG } 00:10:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG "value": [ 00:10:44 DEBUG }, 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG "pod": "ds-cts-1" 00:10:44 DEBUG "instance": "10.142.0.43:9100" 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG "0" 00:10:44 DEBUG [...] 00:10:44 DEBUG } 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG }, 00:10:44 DEBUG "value": [ 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG "0.005" 00:10:44 DEBUG "0.5" 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG }, 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG "value": [ 00:10:44 DEBUG "0" 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG "instance": "10.142.0.43:9100" 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG "12542086144" 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG }, 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG ] 00:10:44 DEBUG "value": [ 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG 1713049844, 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG "0.0020333333333333314" 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:10:44 DEBUG ] 00:10:44 DEBUG } 00:10:44 DEBUG } 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO 00:11:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:11:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713049904" 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG --- status code --- 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713049904" 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713049904" 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 ERROR [http_cmd]: ERROR 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 INFO [loop_until]: kubectl --namespace=xlou top node 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG --- http response --- 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 ERROR --- status code --- 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG --- status code --- 00:11:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG { 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049904" 00:11:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG --- http response --- 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 ERROR --- http response --- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG { 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049904" 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 INFO [loop_until]: OK (rc = 0) 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG { 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG { 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG { 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG "data": { 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG --- stdout --- 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG ----- output ----- 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:11:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG "data": { 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG { 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 7m 1134Mi am-7d597cf7d6-8vr8f 9m 1218Mi am-7d597cf7d6-9wz9h 10m 3366Mi ds-cts-0 312m 473Mi ds-cts-1 256m 479Mi ds-cts-2 249m 475Mi ds-idrepo-0 327m 4277Mi ds-idrepo-1 307m 4263Mi ds-idrepo-2 246m 4111Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 11m 953Mi idm-b85f9d978-rl76f 22m 964Mi lodemon-64cbdcc7b4-vt4kk 1m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 206m 1013Mi 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "data": { 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG { 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG "data": { 00:11:44 DEBUG { 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "data": { 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG --- stderr --- 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG { 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "result": [ 00:11:44 DEBUG 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG { 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "data": { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG "data": { 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:11:44 DEBUG "result": [ 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "result": [ 00:11:44 DEBUG { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "metric": { 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG { 00:11:44 DEBUG "resultType": "vector", 00:11:44 INFO [http_cmd]: http status code OK 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:11:44 DEBUG "data": { 00:11:44 DEBUG "data": { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG { 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG { 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "result": [ 00:11:44 DEBUG --- status code --- 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:11:44 DEBUG "metric": { 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG }, 00:11:44 DEBUG "metric": { 00:11:44 DEBUG { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG "instance": "10.142.0.107:9100" 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:11:44 DEBUG "metric": { 00:11:44 DEBUG }, 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "instance": "10.142.0.107:9100" 00:11:44 DEBUG { 00:11:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:11:44 DEBUG "instance": "10.142.0.107:9100" 00:11:44 DEBUG "data": { 00:11:44 DEBUG { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG }, 00:11:44 DEBUG "instance": "10.142.0.107:9100" 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG }, 00:11:44 DEBUG }, 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:11:44 DEBUG }, 00:11:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "data": { 00:11:44 DEBUG }, 00:11:44 DEBUG "metric": { 00:11:44 DEBUG --- http response --- 00:11:44 DEBUG }, 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "data": { 00:11:44 DEBUG { 00:11:44 DEBUG "value": [ 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG "data": { 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "value": [ 00:11:44 DEBUG }, 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:11:44 DEBUG "metric": { 00:11:44 DEBUG { 00:11:44 DEBUG { 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "metric": { 00:11:44 DEBUG 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG "result": [ 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG ----- output ----- 00:11:44 DEBUG { 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:11:44 DEBUG "result": [ 00:11:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "result": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG { 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "status": "success", 00:11:44 DEBUG "metric": { 00:11:44 DEBUG }, 00:11:44 DEBUG { 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG { 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG "metric": { 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG [...] 00:11:44 DEBUG "data": { 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "value": [ 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "metric": { 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG [...] 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-1" 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "resultType": "vector", 00:11:44 DEBUG "result": [ 00:11:44 DEBUG 00:11:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG { 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "metric": { 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-0" 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG "instance": "10.142.0.43:9100" 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-0" 00:11:44 DEBUG "instance": "10.142.0.43:9100" 00:11:44 DEBUG 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "0" 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-2" 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG "pod": "ds-idrepo-0" 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG ] 00:11:44 DEBUG }, 00:11:44 DEBUG "0.03333333333333333" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "pod": "ds-idrepo-0" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG } 00:11:44 DEBUG "value": [ 00:11:44 DEBUG ] 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG }, 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG } 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG } 00:11:44 DEBUG "0" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG "0" 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG "0" 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG "pod": "ds-cts-1" 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "instance": "10.142.0.107:9100" 00:11:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:11:44 DEBUG 00:11:44 DEBUG "0.2049723701616129" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-2" 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG 00:11:44 DEBUG "instance": "10.142.0.43:9100" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "instance": "10.142.0.43:9100" 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG }, 00:11:44 DEBUG }, 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG "5286.4" 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG "pod": "ds-idrepo-2" 00:11:44 DEBUG }, 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG "12564517888" 00:11:44 DEBUG "pod": "ds-idrepo-2" 00:11:44 DEBUG }, 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG }, 00:11:44 DEBUG "0" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "value": [ 00:11:44 DEBUG [...] 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG [...] 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG }, 00:11:44 DEBUG ] 00:11:44 DEBUG }, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "0" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG "0" 00:11:44 DEBUG "value": [ 00:11:44 DEBUG } 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG ] 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG "0" 00:11:44 DEBUG ] 00:11:44 DEBUG "1161588736" 00:11:44 DEBUG 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG "pod": "ds-idrepo-0" 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG "0" 00:11:44 DEBUG 00:11:44 DEBUG 00:11:44 DEBUG [...] 00:11:44 DEBUG 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG ] 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG "10925.599999999999" 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG "26838.538724546357" 00:11:44 DEBUG ] 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG "0" 00:11:44 DEBUG } 00:11:44 DEBUG "5947.176194643795" 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG }, 00:11:44 DEBUG }, 00:11:44 DEBUG } 00:11:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG }, 00:11:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "value": [ 00:11:44 DEBUG } 00:11:44 DEBUG }, 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG "value": [ 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG ] 00:11:44 DEBUG "pod": "ds-cts-1" 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG [...] 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG "value": [ 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG }, 00:11:44 DEBUG }, 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG } 00:11:44 DEBUG "0" 00:11:44 DEBUG "1" 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG } 00:11:44 DEBUG "0.03333333333333333" 00:11:44 DEBUG } 00:11:44 DEBUG "value": [ 00:11:44 DEBUG "value": [ 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG "0.005" 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG "0.23183583254994272" 00:11:44 DEBUG } 00:11:44 DEBUG 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG ] 00:11:44 DEBUG "instance": "10.142.0.43:9100" 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG "0" 00:11:44 DEBUG } 00:11:44 DEBUG }, 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG ] 00:11:44 DEBUG "value": [ 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG 1713049904, 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG "0.0005999999999997859" 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 DEBUG ] 00:11:44 DEBUG } 00:11:44 DEBUG } 00:11:44 INFO [loop_until]: OK (rc = 0) 00:11:44 DEBUG --- stdout --- 00:11:44 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 86m 0% 4819Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 336m 1% 2424Mi 2% gke-xlou-cdm-default-pool-1b0769c8-nhlf 51m 0% 3197Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 105m 0% 2978Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 337m 1% 6200Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 625m 2% 3873Mi 3% gke-xlou-cdm-default-pool-1b0769c8-xnmp 71m 0% 2891Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 61m 0% 2687Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 334m 1% 5398Mi 4% gke-xlou-cdm-ds-0f601751-j488 335m 1% 1800Mi 1% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2223Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 85m 0% 4915Mi 4% gke-xlou-cdm-ds-0f601751-nmxm 595m 1% 5630Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 185m 1% 4863Mi 8% 00:11:44 DEBUG --- stderr --- 00:11:44 DEBUG 00:12:44 INFO 00:12:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:12:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049964" 00:12:44 ERROR [http_cmd]: ERROR 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 ERROR --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG --- status code --- 00:12:44 ERROR --- http response --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG { 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG ----- output ----- 00:12:44 INFO [loop_until]: OK (rc = 0) 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG { 00:12:44 DEBUG --- stdout --- 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG { 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 5m 1135Mi am-7d597cf7d6-8vr8f 6m 1218Mi am-7d597cf7d6-9wz9h 5m 3366Mi ds-cts-0 95m 473Mi ds-cts-1 7m 479Mi ds-cts-2 6m 476Mi ds-idrepo-0 94m 4181Mi ds-idrepo-1 9m 4264Mi ds-idrepo-2 32m 4111Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 954Mi idm-b85f9d978-rl76f 7m 964Mi lodemon-64cbdcc7b4-vt4kk 2m 62Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 185m 1013Mi 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "data": { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "data": { 00:12:44 DEBUG "data": { 00:12:44 DEBUG --- stderr --- 00:12:44 DEBUG "data": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "data": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "data": { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:12:44 DEBUG { 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG }, 00:12:44 DEBUG "metric": { 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG "instance": "10.142.0.107:9100" 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG "instance": "10.142.0.107:9100" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG "instance": "10.142.0.107:9100" 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG }, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG 00:12:44 DEBUG "pod": "ds-cts-1" 00:12:44 DEBUG "pod": "ds-idrepo-0" 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG "pod": "ds-idrepo-0" 00:12:44 DEBUG "instance": "10.142.0.43:9100" 00:12:44 DEBUG 00:12:44 DEBUG "instance": "10.142.0.43:9100" 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG "pod": "ds-cts-1" 00:12:44 DEBUG "instance": "10.142.0.43:9100" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "0.0029658245964080295" 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG ] 00:12:44 DEBUG }, 00:12:44 DEBUG 1713049964, 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049964" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "value": [ 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG } 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "0" 00:12:44 DEBUG "16631.18472574229" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713049964" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "0" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713049964" 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713049964" 00:12:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713049964" 00:12:44 DEBUG ] 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG "1" 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG "0" 00:12:44 DEBUG "0" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG ] 00:12:44 DEBUG --- status code --- 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG "11025.199999999997" 00:12:44 DEBUG "0" 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG } 00:12:44 DEBUG "500.18600678924173" 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG "12565527552" 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG --- status code --- 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG --- status code --- 00:12:44 INFO [loop_until]: kubectl --namespace=xlou top node 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG "0" 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG --- http response --- 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG } 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:12:44 INFO [http_cmd]: http status code OK 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG ] 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG --- status code --- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG } 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG } 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG } 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG { 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG { 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG } 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG } 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG { 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG } 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG --- http response --- 00:12:44 DEBUG { 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG { 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG "data": { 00:12:44 DEBUG } 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG { 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG "data": { 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG ----- output ----- 00:12:44 DEBUG "data": { 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "data": { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "status": "success", 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG "data": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "data": { 00:12:44 DEBUG { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG { 00:12:44 DEBUG { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG { 00:12:44 DEBUG "resultType": "vector", 00:12:44 DEBUG "metric": { 00:12:44 DEBUG { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "metric": { 00:12:44 DEBUG { 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "result": [ 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG { 00:12:44 DEBUG }, 00:12:44 DEBUG { 00:12:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:12:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG { 00:12:44 DEBUG }, 00:12:44 DEBUG "instance": "10.142.0.107:9100" 00:12:44 DEBUG }, 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "metric": { 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG "metric": { 00:12:44 DEBUG "value": [ 00:12:44 DEBUG }, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "instance": "10.142.0.107:9100" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "pod": "ds-idrepo-1" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "pod": "ds-idrepo-0" 00:12:44 DEBUG "pod": "ds-idrepo-0" 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG [...] 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG 00:12:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG 00:12:44 DEBUG }, 00:12:44 DEBUG "instance": "10.142.0.43:9100" 00:12:44 DEBUG }, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "instance": "10.142.0.43:9100" 00:12:44 DEBUG }, 00:12:44 DEBUG }, 00:12:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG }, 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 00:12:44 DEBUG "value": [ 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "value": [ 00:12:44 DEBUG }, 00:12:44 DEBUG "0" 00:12:44 DEBUG "0" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "0" 00:12:44 DEBUG "0" 00:12:44 DEBUG }, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG "value": [ 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG }, 00:12:44 DEBUG "0" 00:12:44 DEBUG "pod": "ds-idrepo-0" 00:12:44 DEBUG "5246.166666666666" 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG "value": [ 00:12:44 DEBUG "0.00040000000000001514" 00:12:44 DEBUG "1228046336" 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG "0" 00:12:44 DEBUG "value": [ 00:12:44 DEBUG ] 00:12:44 DEBUG }, 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG "0.22314026396031353" 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG } 00:12:44 DEBUG "value": [ 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG "0" 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG "0.005" 00:12:44 DEBUG ] 00:12:44 DEBUG 1713049964, 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG "0" 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG ] 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 DEBUG } 00:12:44 INFO [loop_until]: OK (rc = 0) 00:12:44 DEBUG --- stdout --- 00:12:44 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 83m 0% 4815Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 61m 0% 2437Mi 2% gke-xlou-cdm-default-pool-1b0769c8-nhlf 55m 0% 3199Mi 2% gke-xlou-cdm-default-pool-1b0769c8-qkns 85m 0% 2971Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 64m 0% 6220Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 158m 0% 3919Mi 3% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 58m 0% 2688Mi 2% gke-xlou-cdm-ds-0f601751-hmm7 95m 0% 5421Mi 4% gke-xlou-cdm-ds-0f601751-j488 56m 0% 1792Mi 1% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2226Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 80m 0% 4915Mi 4% gke-xlou-cdm-ds-0f601751-nmxm 65m 0% 5522Mi 4% gke-xlou-cdm-frontend-a8771548-i82n 291m 1% 4769Mi 8% 00:12:44 DEBUG --- stderr --- 00:12:44 DEBUG 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050024" 00:13:44 INFO 00:13:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:13:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:13:44 ERROR [http_cmd]: ERROR 00:13:44 ERROR --- status code --- 00:13:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:13:44 ERROR --- http response --- 00:13:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:13:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "data": { 00:13:44 DEBUG "data": { 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG "data": { 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "result": [ 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "result": [ 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG { 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "data": { 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "result": [ 00:13:44 DEBUG [...] 00:13:44 DEBUG [...] 00:13:44 DEBUG [...] 00:13:44 DEBUG { 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "instance": "10.142.0.107:9100" 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG "value": [ 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 1713050024, 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 00:13:44 DEBUG "0" 00:13:44 DEBUG "0" 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "0" 00:13:44 DEBUG [...] 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG ] 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG "instance": "10.142.0.43:9100" 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG { 00:13:44 DEBUG }, 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "data": { 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "0.0006666666666666673" 00:13:44 DEBUG "result": [ 00:13:44 DEBUG ] 00:13:44 DEBUG { 00:13:44 DEBUG } 00:13:44 DEBUG "metric": { 00:13:44 DEBUG ] 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG } 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG }, 00:13:44 DEBUG } 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "value": [ 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG [...] 00:13:44 DEBUG { 00:13:44 DEBUG 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG 00:13:44 DEBUG "data": { 00:13:44 DEBUG 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "pod": "ds-cts-1" 00:13:44 DEBUG "result": [ 00:13:44 DEBUG }, 00:13:44 DEBUG { 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:13:44 DEBUG "0" 00:13:44 DEBUG }, 00:13:44 DEBUG ] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG [...] 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG }, 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "359697.06666666665" 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG { 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "data": { 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG "result": [ 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "data": { 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG }, 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG { 00:13:44 DEBUG 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG [...] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG [...] 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 00:13:44 DEBUG "837.1666666666666" 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG ] 00:13:44 DEBUG }, 00:13:44 DEBUG } 00:13:44 DEBUG "value": [ 00:13:44 DEBUG ] 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG } 00:13:44 DEBUG "768.345123888889" 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050024" 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG --- status code --- 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG --- http response --- 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG { 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG { 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG { 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG { 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG "data": { 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "data": { 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "data": { 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "data": { 00:13:44 DEBUG "result": [ 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG "result": [ 00:13:44 DEBUG { 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG { 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG "metric": { 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG { 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG "data": { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG }, 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG { 00:13:44 DEBUG }, 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "result": [ 00:13:44 DEBUG }, 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "data": { 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG { 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "data": { 00:13:44 DEBUG 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG "result": [ 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG 00:13:44 DEBUG "result": [ 00:13:44 DEBUG 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG { 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG 00:13:44 DEBUG [...] 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG { 00:13:44 DEBUG [...] 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "metric": { 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG [...] 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG --- http response --- 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 DEBUG [...] 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050024" 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG ----- output ----- 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050024" 00:13:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050024" 00:13:44 DEBUG 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG [...] 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG "data": { 00:13:44 DEBUG "data": { 00:13:44 DEBUG }, 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG [...] 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "768.8572383333333" 00:13:44 DEBUG 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG [...] 00:13:44 DEBUG "0.005" 00:13:44 DEBUG 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "result": [ 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG ] 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG "0" 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG 00:13:44 DEBUG { 00:13:44 DEBUG { 00:13:44 DEBUG "0" 00:13:44 DEBUG } 00:13:44 DEBUG }, 00:13:44 DEBUG ] 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG } 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG "pod": "ds-cts-1" 00:13:44 DEBUG "instance": "10.142.0.107:9100" 00:13:44 DEBUG "instance": "10.142.0.107:9100" 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG ] 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG } 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG "value": [ 00:13:44 DEBUG } 00:13:44 DEBUG "0" 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG } 00:13:44 DEBUG --- http response --- 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG }, 00:13:44 DEBUG } 00:13:44 DEBUG "value": [ 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG "value": [ 00:13:44 DEBUG ] 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG } 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "0" 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG { 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG "12539" 00:13:44 DEBUG ] 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG 00:13:44 DEBUG [...] 00:13:44 DEBUG } 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG ] 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG } 00:13:44 DEBUG [...] 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG "data": { 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG } 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG ] 00:13:44 DEBUG 00:13:44 DEBUG ----- output ----- 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG 00:13:44 DEBUG "resultType": "vector", 00:13:44 INFO [http_cmd]: http status code OK 00:13:44 DEBUG { 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG ] 00:13:44 DEBUG { 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG { 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG 00:13:44 DEBUG "result": [ 00:13:44 DEBUG --- status code --- 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG } 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "instance": "10.142.0.43:9100" 00:13:44 DEBUG { 00:13:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:13:44 DEBUG "data": { 00:13:44 DEBUG } 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG "data": { 00:13:44 DEBUG "instance": "10.142.0.43:9100" 00:13:44 DEBUG "data": { 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG }, 00:13:44 DEBUG "metric": { 00:13:44 DEBUG --- http response --- 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG { 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG }, 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:13:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:13:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG { 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG }, 00:13:44 DEBUG ----- output ----- 00:13:44 DEBUG { 00:13:44 DEBUG "data": { 00:13:44 DEBUG { 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG { 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG { 00:13:44 DEBUG "value": [ 00:13:44 DEBUG { 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "12135.33333333333" 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "12566111232" 00:13:44 DEBUG "data": { 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG 00:13:44 DEBUG "status": "success", 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "instance": "10.142.0.107:9100" 00:13:44 DEBUG ] 00:13:44 DEBUG "instance": "10.142.0.107:9100" 00:13:44 DEBUG ] 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "data": { 00:13:44 DEBUG 00:13:44 DEBUG "data": { 00:13:44 DEBUG }, 00:13:44 DEBUG { 00:13:44 DEBUG }, 00:13:44 DEBUG } 00:13:44 DEBUG }, 00:13:44 DEBUG } 00:13:44 DEBUG "result": [ 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG 00:13:44 DEBUG "resultType": "vector", 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "metric": { 00:13:44 DEBUG "value": [ 00:13:44 DEBUG ] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG ] 00:13:44 DEBUG { 00:13:44 DEBUG "result": [ 00:13:44 DEBUG [...] 00:13:44 DEBUG "result": [ 00:13:44 DEBUG 00:13:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG "metric": { 00:13:44 DEBUG { 00:13:44 DEBUG 00:13:44 DEBUG { 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 00:13:44 DEBUG "metric": { 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG "pod": "ds-idrepo-1" 00:13:44 DEBUG 00:13:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:13:44 DEBUG [...] 00:13:44 DEBUG 00:13:44 DEBUG [...] 00:13:44 DEBUG [...] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG }, 00:13:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG }, 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG [...] 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG 00:13:44 DEBUG "instance": "10.142.0.43:9100" 00:13:44 DEBUG "instance": "10.142.0.43:9100" 00:13:44 DEBUG [...] 00:13:44 DEBUG 00:13:44 DEBUG "1624104960" 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG 00:13:44 DEBUG [...] 00:13:44 DEBUG ] 00:13:44 DEBUG [...] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG 00:13:44 DEBUG "0" 00:13:44 DEBUG }, 00:13:44 DEBUG "0.0004666666666667633" 00:13:44 DEBUG "6926.766666666666" 00:13:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:13:44 DEBUG 00:13:44 DEBUG } 00:13:44 DEBUG 00:13:44 DEBUG ] 00:13:44 DEBUG "value": [ 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG }, 00:13:44 DEBUG "pod": "ds-idrepo-0" 00:13:44 DEBUG } 00:13:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:13:44 DEBUG } 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG "value": [ 00:13:44 DEBUG }, 00:13:44 DEBUG }, 00:13:44 DEBUG ] 00:13:44 DEBUG "2.5403409830488313" 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG "value": [ 00:13:44 DEBUG "value": [ 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG "0" 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG 1713050024, 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG "0" 00:13:44 DEBUG "0" 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG ] 00:13:44 DEBUG ] 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 DEBUG } 00:13:44 INFO 00:13:44 INFO [loop_until]: kubectl --namespace=xlou top node 00:13:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:13:44 INFO [loop_until]: OK (rc = 0) 00:13:44 DEBUG --- stdout --- 00:13:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 3672m 3809Mi am-7d597cf7d6-8vr8f 5722m 4003Mi am-7d597cf7d6-9wz9h 5538m 3827Mi ds-cts-0 3615m 1115Mi ds-cts-1 3700m 1286Mi ds-cts-2 3368m 1203Mi ds-idrepo-0 2684m 4203Mi ds-idrepo-1 789m 4272Mi ds-idrepo-2 742m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 955Mi idm-b85f9d978-rl76f 9m 965Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 3244m 1462Mi 00:13:44 DEBUG --- stderr --- 00:13:44 DEBUG 00:13:44 INFO [loop_until]: OK (rc = 0) 00:13:44 DEBUG --- stdout --- 00:13:44 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 84m 0% 4817Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 3698m 12% 3222Mi 2% gke-xlou-cdm-default-pool-1b0769c8-nhlf 6584m 22% 5966Mi 5% gke-xlou-cdm-default-pool-1b0769c8-qkns 5805m 19% 2973Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1104m 3% 6276Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 3793m 12% 4568Mi 4% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2872Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 6798m 22% 5438Mi 4% gke-xlou-cdm-ds-0f601751-hmm7 1052m 3% 5460Mi 4% gke-xlou-cdm-ds-0f601751-j488 3885m 13% 2649Mi 2% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2237Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 8071m 27% 5464Mi 4% gke-xlou-cdm-ds-0f601751-nmxm 6426m 21% 5575Mi 4% gke-xlou-cdm-frontend-a8771548-i82n 3996m 25% 5252Mi 8% 00:13:44 DEBUG --- stderr --- 00:13:44 DEBUG 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050084" 00:14:44 ERROR [http_cmd]: ERROR 00:14:44 ERROR --- status code --- 00:14:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:14:44 ERROR --- http response --- 00:14:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:14:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "ds-cts-1" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "2115.2" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "instance": "10.142.0.107:9100" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "instance": "10.142.0.43:9100" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:14:44 DEBUG { 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG { 00:14:44 DEBUG "data": { 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "data": { 00:14:44 DEBUG "result": [ 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG "result": [ 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG { 00:14:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG }, 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:14:44 DEBUG }, 00:14:44 DEBUG }, 00:14:44 DEBUG 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "idm-b85f9d978-rl76f" 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG }, 00:14:44 DEBUG 00:14:44 DEBUG "pod": "ds-idrepo-0" 00:14:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "pod": "idm-b85f9d978-rl76f" 00:14:44 DEBUG }, 00:14:44 DEBUG }, 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "0" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG ] 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG "0" 00:14:44 DEBUG } 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG ] 00:14:44 DEBUG ] 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "2115.5666666666666" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "ds-idrepo-0" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:14:44 DEBUG { 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG { 00:14:44 DEBUG "data": { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "result": [ 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG "resultType": "vector", 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG { 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "result": [ 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG "metric": { 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG { 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG "metric": { 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG }, 00:14:44 DEBUG "instance": "10.142.0.107:9100" 00:14:44 DEBUG }, 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG "value": [ 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG }, 00:14:44 DEBUG 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG { 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG "data": { 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "resultType": "vector", 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG "result": [ 00:14:44 DEBUG [...] 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG { 00:14:44 DEBUG 00:14:44 DEBUG --- http response --- 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:14:44 DEBUG "metric": { 00:14:44 DEBUG 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG "pod": "ds-idrepo-0" 00:14:44 DEBUG 00:14:44 DEBUG }, 00:14:44 DEBUG "instance": "10.142.0.107:9100" 00:14:44 DEBUG 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG }, 00:14:44 DEBUG "pod": "ds-idrepo-0" 00:14:44 DEBUG "value": [ 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 DEBUG }, 00:14:44 DEBUG "instance": "10.142.0.43:9100" 00:14:44 DEBUG { 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG "value": [ 00:14:44 DEBUG }, 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG }, 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "1638047744" 00:14:44 DEBUG 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "data": { 00:14:44 DEBUG "0" 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG ] 00:14:44 DEBUG 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG ] 00:14:44 DEBUG { 00:14:44 DEBUG "0" 00:14:44 DEBUG } 00:14:44 DEBUG 00:14:44 DEBUG "5278.133333333333" 00:14:44 DEBUG "result": [ 00:14:44 DEBUG } 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG ] 00:14:44 DEBUG ] 00:14:44 DEBUG [...] 00:14:44 DEBUG ] 00:14:44 DEBUG { 00:14:44 DEBUG ] 00:14:44 DEBUG "data": { 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG 00:14:44 DEBUG } 00:14:44 DEBUG "metric": { 00:14:44 DEBUG } 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG 00:14:44 DEBUG ] 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG } 00:14:44 DEBUG "result": [ 00:14:44 DEBUG } 00:14:44 DEBUG 00:14:44 DEBUG } 00:14:44 DEBUG }, 00:14:44 DEBUG { 00:14:44 DEBUG } 00:14:44 DEBUG "instance": "10.142.0.43:9100" 00:14:44 DEBUG } 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "metric": { 00:14:44 DEBUG }, 00:14:44 DEBUG 00:14:44 DEBUG "instance": "10.142.0.107:9100" 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG }, 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG 00:14:44 DEBUG "value": [ 00:14:44 DEBUG "12166.599999999999" 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG ] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG } 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG ] 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG } 00:14:44 DEBUG "pod": "ds-idrepo-0" 00:14:44 DEBUG 00:14:44 DEBUG } 00:14:44 DEBUG }, 00:14:44 DEBUG 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "instance": "10.142.0.43:9100" 00:14:44 DEBUG "0" 00:14:44 DEBUG }, 00:14:44 DEBUG ] 00:14:44 DEBUG "value": [ 00:14:44 DEBUG } 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG ] 00:14:44 DEBUG "0.00026666666666651885" 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "2115.4666666666667" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0.005" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "ds-idrepo-1" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "ds-cts-1" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050084" 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050084" 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "119010" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "instance": "10.142.0.107:9100" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "instance": "10.142.0.43:9100" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "12566680576" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "5.979110239425316" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO 00:14:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:14:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:14:44 INFO [http_cmd]: http status code OK 00:14:44 DEBUG --- status code --- 00:14:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:14:44 DEBUG --- http response --- 00:14:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:14:44 DEBUG ----- output ----- 00:14:44 DEBUG { 00:14:44 DEBUG "status": "success", 00:14:44 DEBUG "data": { 00:14:44 DEBUG "resultType": "vector", 00:14:44 DEBUG "result": [ 00:14:44 DEBUG { 00:14:44 DEBUG "metric": { 00:14:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG [...] 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG 00:14:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:14:44 DEBUG }, 00:14:44 DEBUG "value": [ 00:14:44 DEBUG 1713050084, 00:14:44 DEBUG "0" 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG ] 00:14:44 DEBUG } 00:14:44 DEBUG } 00:14:44 INFO [loop_until]: OK (rc = 0) 00:14:44 DEBUG --- stdout --- 00:14:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 8878m 4845Mi am-7d597cf7d6-8vr8f 8181m 4952Mi am-7d597cf7d6-9wz9h 10859m 4751Mi ds-cts-0 4545m 2427Mi ds-cts-1 4784m 2806Mi ds-cts-2 4722m 2830Mi ds-idrepo-0 10671m 4216Mi ds-idrepo-1 1639m 4272Mi ds-idrepo-2 1563m 4088Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 955Mi idm-b85f9d978-rl76f 6m 966Mi lodemon-64cbdcc7b4-vt4kk 8m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5960m 1471Mi 00:14:44 DEBUG --- stderr --- 00:14:44 DEBUG 00:14:44 INFO 00:14:44 INFO [loop_until]: kubectl --namespace=xlou top node 00:14:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:14:45 INFO [loop_until]: OK (rc = 0) 00:14:45 DEBUG --- stdout --- 00:14:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 82m 0% 4822Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4726m 15% 4887Mi 4% gke-xlou-cdm-default-pool-1b0769c8-nhlf 9008m 30% 6866Mi 6% gke-xlou-cdm-default-pool-1b0769c8-qkns 15556m 52% 2973Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1760m 5% 6272Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4643m 15% 6116Mi 5% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2873Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 8719m 29% 6369Mi 5% gke-xlou-cdm-ds-0f601751-hmm7 1667m 5% 5412Mi 4% gke-xlou-cdm-ds-0f601751-j488 4856m 16% 4364Mi 3% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2230Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 11747m 39% 6241Mi 5% gke-xlou-cdm-ds-0f601751-nmxm 10055m 33% 5595Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6108m 38% 5253Mi 8% 00:14:45 DEBUG --- stderr --- 00:14:45 DEBUG 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 ERROR [http_cmd]: ERROR 00:15:44 ERROR --- status code --- 00:15:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:15:44 ERROR --- http response --- 00:15:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:15:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-cts-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "0" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "4269.863504576994" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "5203.896146203211" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "0" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "0" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG }, 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG "value": [ 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG 00:15:44 DEBUG { 00:15:44 DEBUG 00:15:44 DEBUG { 00:15:44 DEBUG 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG [...] 00:15:44 DEBUG "data": { 00:15:44 DEBUG [...] 00:15:44 DEBUG "data": { 00:15:44 DEBUG 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG 00:15:44 DEBUG "resultType": "vector", 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG 00:15:44 DEBUG "result": [ 00:15:44 DEBUG 00:15:44 DEBUG "result": [ 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG 00:15:44 DEBUG { 00:15:44 DEBUG 00:15:44 DEBUG { 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG "metric": { 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG }, 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "instance": "10.142.0.107:9100" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG }, 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "0" 00:15:44 DEBUG { 00:15:44 DEBUG 00:15:44 DEBUG "2103.866666666667" 00:15:44 DEBUG 00:15:44 DEBUG ] 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG 00:15:44 DEBUG ] 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG "data": { 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG ] 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG [...] 00:15:44 DEBUG ] 00:15:44 DEBUG [...] 00:15:44 DEBUG } 00:15:44 DEBUG "result": [ 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG { 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG "metric": { 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "instance": "10.142.0.107:9100" 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG "instance": "10.142.0.43:9100" 00:15:44 DEBUG }, 00:15:44 DEBUG }, 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "value": [ 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG --- http response --- 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG 00:15:44 DEBUG "0" 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:15:44 DEBUG "6796.5" 00:15:44 DEBUG 00:15:44 DEBUG ] 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG ] 00:15:44 DEBUG [...] 00:15:44 DEBUG } 00:15:44 DEBUG { 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG ] 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG ] 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG "data": { 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG } 00:15:44 DEBUG "instance": "10.142.0.43:9100" 00:15:44 DEBUG "result": [ 00:15:44 DEBUG }, 00:15:44 DEBUG { 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "metric": { 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:15:44 DEBUG "0.00033333333333333365" 00:15:44 DEBUG }, 00:15:44 DEBUG ] 00:15:44 DEBUG "value": [ 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG ] 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG } 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "10.529257093527042" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "2103.3333333333335" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050144" 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG --- http response --- 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG { 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG { 00:15:44 DEBUG { 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG { 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG { 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG { 00:15:44 DEBUG { 00:15:44 DEBUG "data": { 00:15:44 DEBUG { 00:15:44 DEBUG "data": { 00:15:44 DEBUG "data": { 00:15:44 DEBUG { 00:15:44 DEBUG "data": { 00:15:44 DEBUG "status": "success", 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "data": { 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG "data": { 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG "data": { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "data": { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "data": { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG { 00:15:44 DEBUG { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "metric": { 00:15:44 DEBUG { 00:15:44 DEBUG { 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:15:44 DEBUG "data": { 00:15:44 DEBUG { 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG { 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:15:44 DEBUG { 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "metric": { 00:15:44 DEBUG }, 00:15:44 DEBUG "metric": { 00:15:44 DEBUG }, 00:15:44 DEBUG }, 00:15:44 DEBUG "metric": { 00:15:44 DEBUG }, 00:15:44 DEBUG "instance": "10.142.0.107:9100" 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG { 00:15:44 DEBUG "result": [ 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "instance": "10.142.0.107:9100" 00:15:44 DEBUG "value": [ 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG }, 00:15:44 DEBUG }, 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG { 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "data": { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "result": [ 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG { 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG [...] 00:15:44 DEBUG "metric": { 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "instance": "10.142.0.107:9100" 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:15:44 DEBUG 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "value": [ 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG "instance": "10.142.0.43:9100" 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "instance": "10.142.0.43:9100" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG }, 00:15:44 DEBUG }, 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG }, 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG }, 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "245549" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "0.0055" 00:15:44 DEBUG "1640267776" 00:15:44 DEBUG "value": [ 00:15:44 DEBUG "0" 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG [...] 00:15:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG ] 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG ] 00:15:44 DEBUG "0.00036666666666675953" 00:15:44 DEBUG "0" 00:15:44 DEBUG 00:15:44 DEBUG }, 00:15:44 DEBUG "0" 00:15:44 DEBUG } 00:15:44 DEBUG "12336.999999999998" 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG "5.9472040255576895" 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG 00:15:44 DEBUG "value": [ 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG ] 00:15:44 DEBUG "instance": "10.142.0.43:9100" 00:15:44 DEBUG "2103.866666666667" 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG }, 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG "value": [ 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG } 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG ] 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:15:44 DEBUG } 00:15:44 DEBUG "12567145472" 00:15:44 DEBUG } 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG { 00:15:44 DEBUG } 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG ] 00:15:44 DEBUG "data": { 00:15:44 DEBUG } 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG } 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "ds-idrepo-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-cts-1" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "0" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050144" 00:15:44 INFO [http_cmd]: http status code OK 00:15:44 DEBUG --- status code --- 00:15:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:15:44 DEBUG --- http response --- 00:15:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:15:44 DEBUG ----- output ----- 00:15:44 DEBUG { 00:15:44 DEBUG "status": "success", 00:15:44 DEBUG "data": { 00:15:44 DEBUG "resultType": "vector", 00:15:44 DEBUG "result": [ 00:15:44 DEBUG { 00:15:44 DEBUG "metric": { 00:15:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG [...] 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG 00:15:44 DEBUG "pod": "ds-idrepo-0" 00:15:44 DEBUG }, 00:15:44 DEBUG "value": [ 00:15:44 DEBUG 1713050144, 00:15:44 DEBUG "0" 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG ] 00:15:44 DEBUG } 00:15:44 DEBUG } 00:15:44 INFO 00:15:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:15:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:15:44 INFO [loop_until]: OK (rc = 0) 00:15:44 DEBUG --- stdout --- 00:15:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 8845m 4865Mi am-7d597cf7d6-8vr8f 8011m 6025Mi am-7d597cf7d6-9wz9h 11004m 7161Mi ds-cts-0 4413m 3937Mi ds-cts-1 4640m 4388Mi ds-cts-2 4661m 4427Mi ds-idrepo-0 10428m 4216Mi ds-idrepo-1 1664m 4274Mi ds-idrepo-2 1599m 4089Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 955Mi idm-b85f9d978-rl76f 5m 966Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5985m 1472Mi 00:15:44 DEBUG --- stderr --- 00:15:44 DEBUG 00:15:45 INFO 00:15:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:15:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:15:45 INFO [loop_until]: OK (rc = 0) 00:15:45 DEBUG --- stdout --- 00:15:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 83m 0% 4818Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 4659m 15% 6503Mi 5% gke-xlou-cdm-default-pool-1b0769c8-nhlf 8934m 29% 6897Mi 6% gke-xlou-cdm-default-pool-1b0769c8-qkns 15527m 52% 2978Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1768m 5% 6267Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4557m 15% 7612Mi 6% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2885Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 8803m 29% 7437Mi 6% gke-xlou-cdm-ds-0f601751-hmm7 1648m 5% 5404Mi 4% gke-xlou-cdm-ds-0f601751-j488 4781m 16% 5979Mi 5% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2233Mi 2% gke-xlou-cdm-ds-0f601751-nj6q 11517m 38% 8675Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 10406m 34% 5592Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6066m 38% 5252Mi 8% 00:15:45 DEBUG --- stderr --- 00:15:45 DEBUG 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050204" 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-cts-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0.00008752777777777777" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 ERROR [http_cmd]: ERROR 00:16:44 ERROR --- status code --- 00:16:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:16:44 ERROR --- http response --- 00:16:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:16:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "instance": "10.142.0.107:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "instance": "10.142.0.43:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "instance": "10.142.0.107:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "instance": "10.142.0.43:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "5394.099999999999" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "2096" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "2092.7666666666664" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "instance": "10.142.0.107:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "instance": "10.142.0.43:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "12567686144" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-idrepo-0" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "359237" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "1644507136" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "2091.733333333333" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "instance": "10.142.0.107:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "instance": "10.142.0.43:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0.00043333333333350765" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050204" 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0.007" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG [...] 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "766.7385477777777" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "instance": "10.142.0.107:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "instance": "10.142.0.43:9100" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "12308.466666666664" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "ds-idrepo-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "ds-cts-1" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "5.943387282200635" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050204" 00:16:44 INFO [http_cmd]: http status code OK 00:16:44 DEBUG --- status code --- 00:16:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:16:44 DEBUG --- http response --- 00:16:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:16:44 DEBUG ----- output ----- 00:16:44 DEBUG { 00:16:44 DEBUG "status": "success", 00:16:44 DEBUG "data": { 00:16:44 DEBUG "resultType": "vector", 00:16:44 DEBUG "result": [ 00:16:44 DEBUG { 00:16:44 DEBUG "metric": { 00:16:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG [...] 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG 00:16:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:16:44 DEBUG }, 00:16:44 DEBUG "value": [ 00:16:44 DEBUG 1713050204, 00:16:44 DEBUG "0" 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG ] 00:16:44 DEBUG } 00:16:44 DEBUG } 00:16:44 INFO 00:16:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:16:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:16:44 INFO [loop_until]: OK (rc = 0) 00:16:44 DEBUG --- stdout --- 00:16:44 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10411m 7445Mi am-7d597cf7d6-8vr8f 9197m 6131Mi am-7d597cf7d6-9wz9h 12512m 7310Mi ds-cts-0 6390m 5591Mi ds-cts-1 7393m 5997Mi ds-cts-2 7396m 5963Mi ds-idrepo-0 11327m 4217Mi ds-idrepo-1 1724m 4275Mi ds-idrepo-2 1703m 4091Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 11m 955Mi idm-b85f9d978-rl76f 12m 966Mi lodemon-64cbdcc7b4-vt4kk 1m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5903m 1475Mi 00:16:44 DEBUG --- stderr --- 00:16:44 DEBUG 00:16:45 INFO 00:16:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:16:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:16:45 INFO [loop_until]: OK (rc = 0) 00:16:45 DEBUG --- stdout --- 00:16:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 84m 0% 4801Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 7966m 26% 8024Mi 7% gke-xlou-cdm-default-pool-1b0769c8-nhlf 10553m 35% 9466Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 15511m 51% 2979Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1827m 6% 6260Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 7821m 26% 9388Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 72m 0% 2887Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10466m 35% 7562Mi 6% gke-xlou-cdm-ds-0f601751-hmm7 1774m 5% 5423Mi 4% gke-xlou-cdm-ds-0f601751-j488 8524m 28% 7500Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2232Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 13742m 46% 8804Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 11335m 37% 5591Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6047m 38% 5251Mi 8% 00:16:45 DEBUG --- stderr --- 00:16:45 DEBUG 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "5726324.258162691" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 ERROR [http_cmd]: ERROR 00:17:44 ERROR --- status code --- 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 ERROR --- http response --- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG --- http response --- 00:17:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG }, 00:17:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "ds-idrepo-0" 00:17:44 DEBUG 00:17:44 DEBUG "pod": "ds-idrepo-0" 00:17:44 DEBUG }, 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG }, 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG "value": [ 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0" 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG "0" 00:17:44 DEBUG ] 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG "258524956.564888" 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:17:44 DEBUG } 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG ] 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG { 00:17:44 DEBUG } 00:17:44 DEBUG { 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG } 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG } 00:17:44 DEBUG "data": { 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG }, 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "ds-cts-1" 00:17:44 DEBUG "pod": "ds-idrepo-0" 00:17:44 DEBUG }, 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0.00018554166666666667" 00:17:44 DEBUG "0" 00:17:44 DEBUG ] 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "ds-idrepo-0" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "instance": "10.142.0.107:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "instance": "10.142.0.43:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0.00033333333333333365" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "10.968260180621245" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 DEBUG ----- output ----- 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG { 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG "data": { 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG }, 00:17:44 DEBUG { 00:17:44 DEBUG "value": [ 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG 00:17:44 DEBUG "data": { 00:17:44 DEBUG 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG 00:17:44 DEBUG "result": [ 00:17:44 DEBUG [...] 00:17:44 DEBUG { 00:17:44 DEBUG 00:17:44 DEBUG "metric": { 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050264" 00:17:44 DEBUG 00:17:44 DEBUG "instance": "10.142.0.107:9100" 00:17:44 DEBUG 00:17:44 DEBUG }, 00:17:44 DEBUG "pod": "ds-idrepo-0" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG "value": [ 00:17:44 DEBUG }, 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG 00:17:44 DEBUG "value": [ 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG 00:17:44 DEBUG "0" 00:17:44 DEBUG [...] 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG ] 00:17:44 DEBUG 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG 00:17:44 DEBUG { 00:17:44 DEBUG ] 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG } 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG "instance": "10.142.0.43:9100" 00:17:44 DEBUG "data": { 00:17:44 DEBUG } 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG }, 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "value": [ 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG "result": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "5356.966666666666" 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG ] 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG } 00:17:44 DEBUG "data": { 00:17:44 DEBUG }, 00:17:44 DEBUG ] 00:17:44 DEBUG "resultType": "vector", 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG "value": [ 00:17:44 DEBUG } 00:17:44 DEBUG "result": [ 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG { 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG 00:17:44 DEBUG "metric": { 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG [...] 00:17:44 DEBUG }, 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG 00:17:44 DEBUG "value": [ 00:17:44 DEBUG { 00:17:44 DEBUG { 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "data": { 00:17:44 DEBUG "data": { 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG }, 00:17:44 DEBUG [...] 00:17:44 DEBUG "result": [ 00:17:44 DEBUG "result": [ 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG { 00:17:44 DEBUG { 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "379006.5" 00:17:44 DEBUG 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG ] 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG }, 00:17:44 DEBUG }, 00:17:44 DEBUG } 00:17:44 DEBUG }, 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG "value": [ 00:17:44 DEBUG "value": [ 00:17:44 DEBUG ] 00:17:44 DEBUG "value": [ 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG "2097.866666666667" 00:17:44 DEBUG 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG 00:17:44 DEBUG ] 00:17:44 DEBUG [...] 00:17:44 DEBUG { 00:17:44 DEBUG [...] 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG 00:17:44 DEBUG ] 00:17:44 DEBUG 00:17:44 DEBUG "data": { 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG "result": [ 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG }, 00:17:44 DEBUG { 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "instance": "10.142.0.107:9100" 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "2097.8" 00:17:44 DEBUG }, 00:17:44 DEBUG "2098.366666666667" 00:17:44 DEBUG ] 00:17:44 DEBUG "value": [ 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 DEBUG [...] 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "instance": "10.142.0.43:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "12568222720" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:17:44 DEBUG }, 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:17:44 DEBUG "value": [ 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG 00:17:44 DEBUG { 00:17:44 DEBUG 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG 00:17:44 DEBUG "data": { 00:17:44 DEBUG [...] 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG 00:17:44 DEBUG "result": [ 00:17:44 DEBUG 00:17:44 DEBUG { 00:17:44 DEBUG 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:17:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:17:44 DEBUG }, 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG 00:17:44 DEBUG "1647689728" 00:17:44 DEBUG 00:17:44 DEBUG ] 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG [...] 00:17:44 DEBUG ] 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG } 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "instance": "10.142.0.107:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "instance": "10.142.0.43:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0.000500000000000019" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050264" 00:17:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050264" 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "0.008" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "instance": "10.142.0.107:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "instance": "10.142.0.43:9100" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "12162.8" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG { 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "2115.9" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:17:44 INFO [http_cmd]: http status code OK 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG --- status code --- 00:17:44 DEBUG { 00:17:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG --- http response --- 00:17:44 DEBUG "data": { 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG "result": [ 00:17:44 DEBUG { 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "pod": "ds-idrepo-1" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "pod": "ds-cts-1" 00:17:44 DEBUG }, 00:17:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:17:44 DEBUG "value": [ 00:17:44 DEBUG ----- output ----- 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG { 00:17:44 DEBUG "0" 00:17:44 DEBUG "status": "success", 00:17:44 DEBUG ] 00:17:44 DEBUG "data": { 00:17:44 DEBUG } 00:17:44 DEBUG "resultType": "vector", 00:17:44 DEBUG ] 00:17:44 DEBUG "result": [ 00:17:44 DEBUG } 00:17:44 DEBUG { 00:17:44 DEBUG } 00:17:44 DEBUG "metric": { 00:17:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG [...] 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG 00:17:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:17:44 DEBUG }, 00:17:44 DEBUG "value": [ 00:17:44 DEBUG 1713050264, 00:17:44 DEBUG "5.936543821903633" 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG ] 00:17:44 DEBUG } 00:17:44 DEBUG } 00:17:44 INFO 00:17:44 INFO [loop_until]: kubectl --namespace=xlou top pods 00:17:44 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:17:45 INFO [loop_until]: OK (rc = 0) 00:17:45 DEBUG --- stdout --- 00:17:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11039m 7451Mi am-7d597cf7d6-8vr8f 10082m 6150Mi am-7d597cf7d6-9wz9h 13549m 7321Mi ds-cts-0 10253m 6385Mi ds-cts-1 10402m 6494Mi ds-cts-2 10591m 6512Mi ds-idrepo-0 11802m 4218Mi ds-idrepo-1 1852m 4276Mi ds-idrepo-2 1752m 4092Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 955Mi idm-b85f9d978-rl76f 9m 966Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5911m 1477Mi 00:17:45 DEBUG --- stderr --- 00:17:45 DEBUG 00:17:45 INFO 00:17:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:17:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:17:45 INFO [loop_until]: OK (rc = 0) 00:17:45 DEBUG --- stdout --- 00:17:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 4814Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10328m 34% 8536Mi 7% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11340m 37% 9479Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 15519m 51% 2976Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1873m 6% 6251Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10111m 33% 9743Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2890Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11362m 38% 9168Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1846m 6% 5426Mi 4% gke-xlou-cdm-ds-0f601751-j488 10590m 35% 7802Mi 6% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2224Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14733m 49% 8817Mi 7% gke-xlou-cdm-ds-0f601751-nmxm 11894m 39% 5593Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 5770m 36% 5255Mi 8% 00:17:45 DEBUG --- stderr --- 00:17:45 DEBUG 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050324" 00:18:44 ERROR [http_cmd]: ERROR 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 ERROR --- status code --- 00:18:44 DEBUG --- status code --- 00:18:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 ERROR --- http response --- 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:18:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:18:44 DEBUG { 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG { 00:18:44 DEBUG "data": { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "result": [ 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG { 00:18:44 DEBUG "result": [ 00:18:44 DEBUG "metric": { 00:18:44 DEBUG { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG "metric": { 00:18:44 DEBUG }, 00:18:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:18:44 DEBUG "value": [ 00:18:44 DEBUG }, 00:18:44 DEBUG 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-idrepo-0" 00:18:44 DEBUG 00:18:44 DEBUG }, 00:18:44 DEBUG "pod": "idm-b85f9d978-rl76f" 00:18:44 DEBUG "value": [ 00:18:44 DEBUG }, 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG "0" 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG ] 00:18:44 DEBUG "1445.9565202612475" 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-idrepo-0" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "idm-b85f9d978-rl76f" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "4957.565212324277" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-cts-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0.00023333333333333333" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-idrepo-0" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-idrepo-0" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "2108.3" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "instance": "10.142.0.107:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "instance": "10.142.0.43:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "2108.4333333333334" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "idm-b85f9d978-rl76f" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050324" 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "idm-b85f9d978-rl76f" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0.003348363670575666" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-idrepo-0" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "instance": "10.142.0.107:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "instance": "10.142.0.43:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "6834.366666666667" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "378633.5" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "2107.3" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "instance": "10.142.0.107:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "instance": "10.142.0.43:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "12568933376" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "1680322560" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0.008" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "instance": "10.142.0.107:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "instance": "10.142.0.43:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0.0004333333333332708" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050324" 00:18:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050324" 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "instance": "10.142.0.107:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "instance": "10.142.0.43:9100" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "12444.599999999999" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "2102.3" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "5.9587867944283355" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:44 INFO [http_cmd]: http status code OK 00:18:44 DEBUG --- status code --- 00:18:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:18:44 DEBUG --- http response --- 00:18:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:18:44 DEBUG ----- output ----- 00:18:44 DEBUG { 00:18:44 DEBUG "status": "success", 00:18:44 DEBUG "data": { 00:18:44 DEBUG "resultType": "vector", 00:18:44 DEBUG "result": [ 00:18:44 DEBUG { 00:18:44 DEBUG "metric": { 00:18:44 DEBUG "pod": "ds-idrepo-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG [...] 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG 00:18:44 DEBUG "pod": "ds-cts-1" 00:18:44 DEBUG }, 00:18:44 DEBUG "value": [ 00:18:44 DEBUG 1713050324, 00:18:44 DEBUG "0" 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG ] 00:18:44 DEBUG } 00:18:44 DEBUG } 00:18:45 INFO 00:18:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:18:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:18:45 INFO [loop_until]: OK (rc = 0) 00:18:45 DEBUG --- stdout --- 00:18:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10885m 7451Mi am-7d597cf7d6-8vr8f 10458m 7810Mi am-7d597cf7d6-9wz9h 13444m 8979Mi ds-cts-0 10232m 6525Mi ds-cts-1 10088m 6921Mi ds-cts-2 10234m 6501Mi ds-idrepo-0 11758m 4218Mi ds-idrepo-1 1819m 4278Mi ds-idrepo-2 1750m 4092Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 18m 955Mi idm-b85f9d978-rl76f 5m 966Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5943m 1506Mi 00:18:45 DEBUG --- stderr --- 00:18:45 DEBUG 00:18:45 INFO 00:18:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:18:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:18:45 INFO [loop_until]: OK (rc = 0) 00:18:45 DEBUG --- stdout --- 00:18:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4818Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10204m 34% 8533Mi 7% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11455m 38% 9486Mi 8% gke-xlou-cdm-default-pool-1b0769c8-qkns 15552m 52% 2986Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1940m 6% 6264Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10043m 33% 9885Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 72m 0% 2881Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11023m 36% 9230Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1772m 5% 5431Mi 4% gke-xlou-cdm-ds-0f601751-j488 10586m 35% 8266Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2230Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14628m 48% 10492Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11897m 39% 5596Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6091m 38% 5285Mi 9% 00:18:45 DEBUG --- stderr --- 00:18:45 DEBUG 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 ERROR [http_cmd]: ERROR 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 ERROR --- status code --- 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG --- status code --- 00:19:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 ERROR --- http response --- 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:19:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG }, 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "ds-cts-1" 00:19:44 DEBUG "pod": "ds-idrepo-0" 00:19:44 DEBUG }, 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0.000033333333333333335" 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:19:44 DEBUG { 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG { 00:19:44 DEBUG "data": { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "result": [ 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG { 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "result": [ 00:19:44 DEBUG "metric": { 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG { 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG "metric": { 00:19:44 DEBUG }, 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG }, 00:19:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:19:44 DEBUG "value": [ 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG }, 00:19:44 DEBUG 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "ds-idrepo-0" 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG 00:19:44 DEBUG }, 00:19:44 DEBUG "pod": "ds-idrepo-0" 00:19:44 DEBUG }, 00:19:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:19:44 DEBUG "value": [ 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG }, 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG "0" 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "2115.2" 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG ] 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG "0" 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "5292231.700525947" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "ds-idrepo-0" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "9.245221243554232" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "instance": "10.142.0.107:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "instance": "10.142.0.43:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "12569531392" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "2116.0333333333333" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "instance": "10.142.0.107:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "instance": "10.142.0.43:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0.0006666666666666598" 00:19:44 DEBUG ] 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "instance": "10.142.0.107:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "instance": "10.142.0.43:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0.0004333333333332708" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "ds-idrepo-0" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG { 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "378636.5" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "instance": "10.142.0.107:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "instance": "10.142.0.43:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "5268.9" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "2116.5666666666666" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "ds-idrepo-1" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "ds-cts-1" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "1679495168" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "0.008" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050384" 00:19:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050384" 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "instance": "10.142.0.107:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "instance": "10.142.0.43:9100" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "12198.999999999998" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "2095.3" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:44 INFO [http_cmd]: http status code OK 00:19:44 DEBUG --- status code --- 00:19:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:19:44 DEBUG --- http response --- 00:19:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:19:44 DEBUG ----- output ----- 00:19:44 DEBUG { 00:19:44 DEBUG "status": "success", 00:19:44 DEBUG "data": { 00:19:44 DEBUG "resultType": "vector", 00:19:44 DEBUG "result": [ 00:19:44 DEBUG { 00:19:44 DEBUG "metric": { 00:19:44 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG [...] 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG 00:19:44 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:19:44 DEBUG }, 00:19:44 DEBUG "value": [ 00:19:44 DEBUG 1713050384, 00:19:44 DEBUG "5.9191986425628125" 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG ] 00:19:44 DEBUG } 00:19:44 DEBUG } 00:19:45 INFO 00:19:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:19:45 INFO [loop_until]: OK (rc = 0) 00:19:45 DEBUG --- stdout --- 00:19:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11678m 9000Mi am-7d597cf7d6-8vr8f 10408m 7795Mi am-7d597cf7d6-9wz9h 13590m 9094Mi ds-cts-0 10076m 6574Mi ds-cts-1 9727m 6949Mi ds-cts-2 10414m 6660Mi ds-idrepo-0 12177m 4224Mi ds-idrepo-1 1816m 4278Mi ds-idrepo-2 1708m 4094Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 955Mi idm-b85f9d978-rl76f 5m 966Mi lodemon-64cbdcc7b4-vt4kk 8m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6055m 1608Mi 00:19:45 DEBUG --- stderr --- 00:19:45 DEBUG 00:19:45 INFO 00:19:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:19:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:19:45 INFO [loop_until]: OK (rc = 0) 00:19:45 DEBUG --- stdout --- 00:19:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4803Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10333m 34% 8696Mi 7% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11941m 39% 11054Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15679m 52% 2994Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1924m 6% 6259Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10173m 34% 9932Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2883Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11063m 37% 9217Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1736m 5% 5422Mi 4% gke-xlou-cdm-ds-0f601751-j488 10476m 35% 8304Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2229Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14992m 50% 10617Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11994m 40% 5595Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6159m 38% 5383Mi 9% 00:19:45 DEBUG --- stderr --- 00:19:45 DEBUG 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "ds-idrepo-1" 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:44 DEBUG "pod": "ds-idrepo-1" 00:20:44 DEBUG }, 00:20:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "ds-cts-1" 00:20:44 DEBUG 00:20:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:44 DEBUG "pod": "ds-idrepo-0" 00:20:44 DEBUG }, 00:20:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "0.00013333333333333334" 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "2098.7" 00:20:44 DEBUG "0" 00:20:44 DEBUG ] 00:20:44 DEBUG "0" 00:20:44 DEBUG ] 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "ds-idrepo-1" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "ds-idrepo-0" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "0" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 ERROR [http_cmd]: ERROR 00:20:44 ERROR --- status code --- 00:20:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:20:44 ERROR --- http response --- 00:20:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:20:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "ds-idrepo-1" 00:20:44 DEBUG "pod": "ds-idrepo-1" 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "ds-idrepo-0" 00:20:44 DEBUG "pod": "ds-idrepo-0" 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "0" 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG ] 00:20:44 DEBUG "0" 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "0" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "68506.26665981261" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050444" 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "data": { 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG "result": [ 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "instance": "10.142.0.107:9100" 00:20:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:20:44 DEBUG "instance": "10.142.0.107:9100" 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG [...] 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "instance": "10.142.0.43:9100" 00:20:44 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:20:44 DEBUG "instance": "10.142.0.43:9100" 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "12570014720" 00:20:44 DEBUG "11.712138986106265" 00:20:44 DEBUG "0.00040000000000001514" 00:20:44 DEBUG ] 00:20:44 DEBUG ] 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG ] 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG } 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "instance": "10.142.0.107:9100" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "instance": "10.142.0.43:9100" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "0.00033333333333333365" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "2099.1666666666665" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "2099.3333333333335" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050444" 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG --- status code --- 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG --- http response --- 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:44 INFO [http_cmd]: http status code OK 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG --- status code --- 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050444" 00:20:44 DEBUG { 00:20:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG --- http response --- 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "0.008" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:44 DEBUG ----- output ----- 00:20:44 DEBUG { 00:20:44 DEBUG "status": "success", 00:20:44 DEBUG "data": { 00:20:44 DEBUG "resultType": "vector", 00:20:44 DEBUG "result": [ 00:20:44 DEBUG { 00:20:44 DEBUG "metric": { 00:20:44 DEBUG "instance": "10.142.0.107:9100" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG [...] 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG 00:20:44 DEBUG "instance": "10.142.0.43:9100" 00:20:44 DEBUG }, 00:20:44 DEBUG "value": [ 00:20:44 DEBUG 1713050444, 00:20:44 DEBUG "6862.666666666666" 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG ] 00:20:44 DEBUG } 00:20:44 DEBUG } 00:20:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050444" 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "pod": "ds-idrepo-1" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "pod": "ds-idrepo-0" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050444, 00:20:45 DEBUG "0" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050444, 00:20:45 DEBUG "379354" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050444, 00:20:45 DEBUG "0" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050444, 00:20:45 DEBUG "5.921858818448327" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050445" 00:20:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050445" 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "pod": "ds-idrepo-1" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "pod": "ds-cts-1" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050445, 00:20:45 DEBUG "0" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050445, 00:20:45 DEBUG "1786257408" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050445" 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050445, 00:20:45 DEBUG "2125.4666666666667" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050445" 00:20:45 INFO [http_cmd]: http status code OK 00:20:45 DEBUG --- status code --- 00:20:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:20:45 DEBUG --- http response --- 00:20:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:20:45 DEBUG ----- output ----- 00:20:45 DEBUG { 00:20:45 DEBUG "status": "success", 00:20:45 DEBUG "data": { 00:20:45 DEBUG "resultType": "vector", 00:20:45 DEBUG "result": [ 00:20:45 DEBUG { 00:20:45 DEBUG "metric": { 00:20:45 DEBUG "instance": "10.142.0.107:9100" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG [...] 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG 00:20:45 DEBUG "instance": "10.142.0.43:9100" 00:20:45 DEBUG }, 00:20:45 DEBUG "value": [ 00:20:45 DEBUG 1713050445, 00:20:45 DEBUG "12006.133333333331" 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG ] 00:20:45 DEBUG } 00:20:45 DEBUG } 00:20:45 INFO 00:20:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:20:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:20:45 INFO [loop_until]: OK (rc = 0) 00:20:45 DEBUG --- stdout --- 00:20:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11123m 8982Mi am-7d597cf7d6-8vr8f 10333m 7796Mi am-7d597cf7d6-9wz9h 13718m 9105Mi ds-cts-0 10490m 6587Mi ds-cts-1 10468m 6928Mi ds-cts-2 10017m 6922Mi ds-idrepo-0 12111m 4225Mi ds-idrepo-1 1857m 4280Mi ds-idrepo-2 1723m 4096Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 13m 959Mi idm-b85f9d978-rl76f 5m 966Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5977m 1610Mi 00:20:45 DEBUG --- stderr --- 00:20:45 DEBUG 00:20:45 INFO 00:20:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:20:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:20:45 INFO [loop_until]: OK (rc = 0) 00:20:45 DEBUG --- stdout --- 00:20:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4816Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10438m 34% 8950Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11617m 38% 11053Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15574m 52% 2985Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1903m 6% 6282Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10220m 34% 9946Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 77m 0% 2883Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11105m 37% 9214Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1783m 5% 5423Mi 4% gke-xlou-cdm-ds-0f601751-j488 10522m 35% 8293Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2227Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14862m 49% 10636Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11983m 40% 5593Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6018m 37% 5385Mi 9% 00:20:45 DEBUG --- stderr --- 00:20:45 DEBUG 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG --- status code --- 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG --- http response --- 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG { 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG { 00:21:44 DEBUG { 00:21:44 DEBUG { 00:21:44 DEBUG { 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG { 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG "data": { 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:44 DEBUG "data": { 00:21:44 ERROR [http_cmd]: ERROR 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG "data": { 00:21:44 DEBUG "data": { 00:21:44 DEBUG "data": { 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG "data": { 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG "resultType": "vector", 00:21:44 ERROR --- status code --- 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG "result": [ 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG { 00:21:44 DEBUG "result": [ 00:21:44 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:21:44 DEBUG { 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG "result": [ 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG "result": [ 00:21:44 DEBUG "result": [ 00:21:44 DEBUG { 00:21:44 DEBUG "result": [ 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG { 00:21:44 ERROR --- http response --- 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG { 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG { 00:21:44 DEBUG { 00:21:44 DEBUG "metric": { 00:21:44 DEBUG { 00:21:44 DEBUG "data": { 00:21:44 DEBUG "metric": { 00:21:44 DEBUG "data": { 00:21:44 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:21:44 DEBUG "metric": { 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG "metric": { 00:21:44 DEBUG "metric": { 00:21:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:44 DEBUG "metric": { 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:44 DEBUG "resultType": "vector", 00:21:44 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:21:44 DEBUG "pod": "ds-idrepo-1" 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG "pod": "ds-idrepo-1" 00:21:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:21:44 DEBUG }, 00:21:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:21:44 DEBUG "result": [ 00:21:44 DEBUG }, 00:21:44 DEBUG "result": [ 00:21:44 DEBUG }, 00:21:44 DEBUG { 00:21:44 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:21:44 DEBUG }, 00:21:44 DEBUG }, 00:21:44 DEBUG "value": [ 00:21:44 DEBUG }, 00:21:44 DEBUG { 00:21:44 DEBUG "value": [ 00:21:44 DEBUG { 00:21:44 DEBUG "value": [ 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG "value": [ 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 00:21:44 DEBUG "value": [ 00:21:44 DEBUG "metric": { 00:21:44 DEBUG 00:21:44 DEBUG "metric": { 00:21:44 DEBUG 00:21:44 DEBUG "data": { 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG "pod": "ds-idrepo-1" 00:21:44 DEBUG 00:21:44 DEBUG "pod": "ds-idrepo-1" 00:21:44 DEBUG 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG }, 00:21:44 DEBUG 00:21:44 DEBUG }, 00:21:44 DEBUG 00:21:44 DEBUG "result": [ 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG [...] 00:21:44 DEBUG 00:21:44 DEBUG "value": [ 00:21:44 DEBUG [...] 00:21:44 DEBUG "value": [ 00:21:44 DEBUG [...] 00:21:44 DEBUG { 00:21:44 DEBUG [...] 00:21:44 DEBUG [...] 00:21:44 DEBUG 00:21:44 DEBUG [...] 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG "metric": { 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG "pod": "ds-idrepo-1" 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG }, 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:44 DEBUG 00:21:44 DEBUG [...] 00:21:44 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:44 DEBUG [...] 00:21:44 DEBUG "pod": "ds-cts-1" 00:21:44 DEBUG "value": [ 00:21:44 DEBUG "pod": "ds-idrepo-0" 00:21:44 DEBUG "pod": "ds-cts-2" 00:21:44 DEBUG }, 00:21:44 DEBUG "pod": "ds-cts-2" 00:21:44 DEBUG 00:21:44 DEBUG }, 00:21:44 DEBUG 00:21:44 DEBUG }, 00:21:44 DEBUG 00:21:44 DEBUG }, 00:21:44 DEBUG }, 00:21:44 DEBUG "value": [ 00:21:44 DEBUG }, 00:21:44 DEBUG 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 00:21:44 DEBUG "value": [ 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG "0" 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG "pod": "ds-idrepo-0" 00:21:44 DEBUG "2113.3" 00:21:44 DEBUG "pod": "ds-idrepo-0" 00:21:44 DEBUG "0.00013333333333333334" 00:21:44 DEBUG [...] 00:21:44 DEBUG "0" 00:21:44 DEBUG "29818512.520452622" 00:21:44 DEBUG ] 00:21:44 DEBUG "287957.66336767434" 00:21:44 DEBUG }, 00:21:44 DEBUG ] 00:21:44 DEBUG }, 00:21:44 DEBUG ] 00:21:44 DEBUG 00:21:44 DEBUG ] 00:21:44 DEBUG ] 00:21:44 DEBUG } 00:21:44 DEBUG ] 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG 00:21:44 INFO [http_cmd]: http status code OK 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG ] 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG "0" 00:21:44 DEBUG } 00:21:44 DEBUG "0" 00:21:44 DEBUG 00:21:44 DEBUG --- status code --- 00:21:44 DEBUG ] 00:21:44 DEBUG ] 00:21:44 DEBUG } 00:21:44 DEBUG ] 00:21:44 DEBUG ] 00:21:44 DEBUG ] 00:21:44 DEBUG ] 00:21:44 DEBUG ] 00:21:44 DEBUG "pod": "ds-idrepo-0" 00:21:44 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG }, 00:21:44 DEBUG --- http response --- 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG ] 00:21:44 DEBUG } 00:21:44 DEBUG ] 00:21:44 DEBUG "value": [ 00:21:44 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG ----- output ----- 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 DEBUG "0" 00:21:44 DEBUG { 00:21:44 DEBUG ] 00:21:44 DEBUG "status": "success", 00:21:44 DEBUG } 00:21:44 DEBUG "data": { 00:21:44 DEBUG ] 00:21:44 DEBUG "resultType": "vector", 00:21:44 DEBUG } 00:21:44 DEBUG "result": [ 00:21:44 DEBUG } 00:21:44 DEBUG { 00:21:44 DEBUG "metric": { 00:21:44 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:21:44 DEBUG }, 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG [...] 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG 00:21:44 DEBUG "pod": "ds-cts-2" 00:21:44 DEBUG }, 00:21:44 DEBUG "value": [ 00:21:44 DEBUG 1713050504, 00:21:44 DEBUG "7.709842112003891" 00:21:44 DEBUG ] 00:21:44 DEBUG } 00:21:44 DEBUG ] 00:21:44 DEBUG } 00:21:44 DEBUG } 00:21:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050504" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050504, 00:21:45 DEBUG "2113.733333333333" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050505" 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050505" 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "0.008" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "instance": "10.142.0.107:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "instance": "10.142.0.43:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "0.0004666666666667633" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "instance": "10.142.0.107:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "instance": "10.142.0.43:9100" 00:21:45 DEBUG }, 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "12570573824" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "instance": "10.142.0.107:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "instance": "10.142.0.43:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "0" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "2113.7" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050505" 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "1789452288" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "instance": "10.142.0.107:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "instance": "10.142.0.43:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "5518.433333333333" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "ds-idrepo-1" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "ds-idrepo-0" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "0" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "379148.5" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "ds-cts-2" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "0" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "5.920535138047181" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "ds-idrepo-1" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "ds-cts-1" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "0" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "2106.5666666666666" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050505" 00:21:45 INFO [http_cmd]: http status code OK 00:21:45 DEBUG --- status code --- 00:21:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:21:45 DEBUG --- http response --- 00:21:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:21:45 DEBUG ----- output ----- 00:21:45 DEBUG { 00:21:45 DEBUG "status": "success", 00:21:45 DEBUG "data": { 00:21:45 DEBUG "resultType": "vector", 00:21:45 DEBUG "result": [ 00:21:45 DEBUG { 00:21:45 DEBUG "metric": { 00:21:45 DEBUG "instance": "10.142.0.107:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG [...] 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG 00:21:45 DEBUG "instance": "10.142.0.43:9100" 00:21:45 DEBUG }, 00:21:45 DEBUG "value": [ 00:21:45 DEBUG 1713050505, 00:21:45 DEBUG "11951.033333333331" 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG ] 00:21:45 DEBUG } 00:21:45 DEBUG } 00:21:45 INFO 00:21:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:21:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:21:45 INFO [loop_until]: OK (rc = 0) 00:21:45 DEBUG --- stdout --- 00:21:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11168m 8985Mi am-7d597cf7d6-8vr8f 10556m 8868Mi am-7d597cf7d6-9wz9h 13921m 9072Mi ds-cts-0 10255m 6581Mi ds-cts-1 10582m 7056Mi ds-cts-2 10507m 6913Mi ds-idrepo-0 12112m 4226Mi ds-idrepo-1 1895m 4281Mi ds-idrepo-2 1709m 4097Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 959Mi idm-b85f9d978-rl76f 8m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5921m 1611Mi 00:21:45 DEBUG --- stderr --- 00:21:45 DEBUG 00:21:45 INFO 00:21:45 INFO [loop_until]: kubectl --namespace=xlou top node 00:21:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:21:45 INFO [loop_until]: OK (rc = 0) 00:21:45 DEBUG --- stdout --- 00:21:45 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4809Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10473m 35% 8960Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11593m 38% 11042Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15615m 52% 2982Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1861m 6% 6270Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10333m 34% 9997Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2883Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11205m 37% 10287Mi 9% gke-xlou-cdm-ds-0f601751-hmm7 1836m 6% 5427Mi 4% gke-xlou-cdm-ds-0f601751-j488 10542m 35% 8412Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2232Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14885m 49% 10604Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11929m 39% 5602Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 5880m 37% 5384Mi 9% 00:21:45 DEBUG --- stderr --- 00:21:45 DEBUG 00:22:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050564" 00:22:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050564" 00:22:44 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050564" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG --- status code --- 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "ds-cts-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050564, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050564, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 ERROR [http_cmd]: ERROR 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 ERROR --- status code --- 00:22:45 DEBUG --- status code --- 00:22:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 ERROR --- http response --- 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:22:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 DEBUG ----- output ----- 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG { 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG "data": { 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG "result": [ 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG }, 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "value": [ 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG [...] 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG }, 00:22:45 DEBUG 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG }, 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG "value": [ 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "29887.125115406383" 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG ] 00:22:45 DEBUG 00:22:45 DEBUG "pod": "ds-idrepo-0" 00:22:45 DEBUG } 00:22:45 DEBUG "pod": "ds-idrepo-0" 00:22:45 DEBUG }, 00:22:45 DEBUG ] 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG } 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG } 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "ds-idrepo-0" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 DEBUG ----- output ----- 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "2117.2666666666664" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "2117.233333333333" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "ds-idrepo-0" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "5.8736356825803275" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "2117.3" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0.008" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG { 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG "data": { 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "instance": "10.142.0.107:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "instance": "10.142.0.43:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0.00040000000000001514" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "1790185472" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "instance": "10.142.0.107:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "instance": "10.142.0.43:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "12571186176" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "instance": "10.142.0.107:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "instance": "10.142.0.43:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0.00033333333333333365" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "ds-cts-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "instance": "10.142.0.107:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "instance": "10.142.0.43:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "5282.8" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "ds-idrepo-1" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "ds-idrepo-0" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "5.903932008063522" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "379329.5" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "0" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "instance": "10.142.0.107:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "instance": "10.142.0.43:9100" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "12172.666666666664" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050565" 00:22:45 INFO [http_cmd]: http status code OK 00:22:45 DEBUG --- status code --- 00:22:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:22:45 DEBUG --- http response --- 00:22:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:22:45 DEBUG ----- output ----- 00:22:45 DEBUG { 00:22:45 DEBUG "status": "success", 00:22:45 DEBUG "data": { 00:22:45 DEBUG "resultType": "vector", 00:22:45 DEBUG "result": [ 00:22:45 DEBUG { 00:22:45 DEBUG "metric": { 00:22:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG [...] 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG 00:22:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:22:45 DEBUG }, 00:22:45 DEBUG "value": [ 00:22:45 DEBUG 1713050565, 00:22:45 DEBUG "2114.9333333333334" 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG ] 00:22:45 DEBUG } 00:22:45 DEBUG } 00:22:45 INFO 00:22:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:22:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:22:45 INFO [loop_until]: OK (rc = 0) 00:22:45 DEBUG --- stdout --- 00:22:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11035m 8988Mi am-7d597cf7d6-8vr8f 10350m 8364Mi am-7d597cf7d6-9wz9h 13655m 9063Mi ds-cts-0 10165m 6702Mi ds-cts-1 10395m 7054Mi ds-cts-2 10473m 6935Mi ds-idrepo-0 12255m 4227Mi ds-idrepo-1 1789m 4282Mi ds-idrepo-2 1678m 4098Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 959Mi idm-b85f9d978-rl76f 10m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5742m 1614Mi 00:22:45 DEBUG --- stderr --- 00:22:45 DEBUG 00:22:46 INFO 00:22:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:22:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:22:46 INFO [loop_until]: OK (rc = 0) 00:22:46 DEBUG --- stdout --- 00:22:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 4798Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10467m 35% 8981Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11522m 38% 11050Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15625m 52% 2982Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1878m 6% 6260Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10203m 34% 10046Mi 8% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 2888Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11028m 36% 9788Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1826m 6% 5415Mi 4% gke-xlou-cdm-ds-0f601751-j488 10663m 35% 8412Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2221Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14847m 49% 10596Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11571m 38% 5614Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6044m 38% 5384Mi 9% 00:22:46 DEBUG --- stderr --- 00:22:46 DEBUG 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG { 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG "metric": { 00:23:45 ERROR [http_cmd]: ERROR 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG "metric": { 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 ERROR --- status code --- 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG }, 00:23:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG }, 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG "value": [ 00:23:45 ERROR --- http response --- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG "value": [ 00:23:45 DEBUG { 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:23:45 DEBUG 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:23:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:23:45 DEBUG 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG 00:23:45 DEBUG ----- output ----- 00:23:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:23:45 DEBUG 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG "data": { 00:23:45 DEBUG { 00:23:45 DEBUG 00:23:45 DEBUG { 00:23:45 DEBUG 00:23:45 DEBUG { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG [...] 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG [...] 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG "data": { 00:23:45 DEBUG 00:23:45 DEBUG "data": { 00:23:45 DEBUG 00:23:45 DEBUG "data": { 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "result": [ 00:23:45 DEBUG 00:23:45 DEBUG "result": [ 00:23:45 DEBUG 00:23:45 DEBUG "result": [ 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 DEBUG { 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG { 00:23:45 DEBUG "pod": "ds-cts-1" 00:23:45 DEBUG { 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG }, 00:23:45 DEBUG "metric": { 00:23:45 DEBUG }, 00:23:45 DEBUG "metric": { 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG "value": [ 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:23:45 DEBUG "value": [ 00:23:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:23:45 DEBUG "metric": { 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG }, 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG }, 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 DEBUG "0.0001632638888888889" 00:23:45 DEBUG 00:23:45 DEBUG { 00:23:45 DEBUG "value": [ 00:23:45 DEBUG "0" 00:23:45 DEBUG "value": [ 00:23:45 DEBUG }, 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG "value": [ 00:23:45 DEBUG } 00:23:45 DEBUG [...] 00:23:45 DEBUG "data": { 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG "result": [ 00:23:45 DEBUG [...] 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG { 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG "pod": "ds-idrepo-0" 00:23:45 DEBUG "metric": { 00:23:45 DEBUG 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG }, 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 DEBUG 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "value": [ 00:23:45 DEBUG }, 00:23:45 DEBUG "pod": "ds-cts-2" 00:23:45 DEBUG "pod": "ds-cts-2" 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:23:45 DEBUG 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG }, 00:23:45 DEBUG }, 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG "pod": "ds-idrepo-0" 00:23:45 DEBUG "0" 00:23:45 DEBUG 00:23:45 DEBUG "value": [ 00:23:45 DEBUG "value": [ 00:23:45 DEBUG { 00:23:45 DEBUG }, 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "value": [ 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG "48306769.95048453" 00:23:45 DEBUG "97550.47237157873" 00:23:45 DEBUG "data": { 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG ] 00:23:45 DEBUG [...] 00:23:45 DEBUG ] 00:23:45 DEBUG ] 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "0" 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 DEBUG "result": [ 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG ] 00:23:45 DEBUG ] 00:23:45 DEBUG { 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 DEBUG "metric": { 00:23:45 DEBUG ] 00:23:45 DEBUG "pod": "ds-idrepo-0" 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:23:45 DEBUG } 00:23:45 DEBUG }, 00:23:45 DEBUG }, 00:23:45 DEBUG } 00:23:45 DEBUG "value": [ 00:23:45 DEBUG "value": [ 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG 00:23:45 DEBUG "0" 00:23:45 DEBUG 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG [...] 00:23:45 DEBUG ] 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG } 00:23:45 DEBUG 00:23:45 DEBUG "pod": "ds-cts-2" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "10.435889850845998" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "2111.0333333333333" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "2112.3333333333335" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "ds-idrepo-0" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "0" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "instance": "10.142.0.107:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "instance": "10.142.0.43:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "0" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "instance": "10.142.0.107:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "instance": "10.142.0.43:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "12571870208" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "2107.4" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "1792229376" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "0.008" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "ds-idrepo-0" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "0" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "instance": "10.142.0.107:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "instance": "10.142.0.43:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "0.0004666666666667633" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "instance": "10.142.0.107:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "instance": "10.142.0.43:9100" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "6942.066666666667" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "ds-idrepo-1" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "ds-cts-1" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "0" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "5.914292535388687" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050625" 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "379716" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "data": { 00:23:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG }, 00:23:45 DEBUG "result": [ 00:23:45 DEBUG "value": [ 00:23:45 DEBUG { 00:23:45 DEBUG 00:23:45 DEBUG "metric": { 00:23:45 DEBUG 00:23:45 DEBUG "instance": "10.142.0.107:9100" 00:23:45 DEBUG 00:23:45 DEBUG }, 00:23:45 DEBUG [...] 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "ds-cts-2" 00:23:45 DEBUG [...] 00:23:45 DEBUG }, 00:23:45 DEBUG 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG 00:23:45 DEBUG "0" 00:23:45 DEBUG "instance": "10.142.0.43:9100" 00:23:45 DEBUG ] 00:23:45 DEBUG }, 00:23:45 DEBUG } 00:23:45 DEBUG "value": [ 00:23:45 DEBUG ] 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG } 00:23:45 DEBUG "12511.799999999997" 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050625" 00:23:45 INFO [http_cmd]: http status code OK 00:23:45 DEBUG --- status code --- 00:23:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:23:45 DEBUG --- http response --- 00:23:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:23:45 DEBUG ----- output ----- 00:23:45 DEBUG { 00:23:45 DEBUG "status": "success", 00:23:45 DEBUG "data": { 00:23:45 DEBUG "resultType": "vector", 00:23:45 DEBUG "result": [ 00:23:45 DEBUG { 00:23:45 DEBUG "metric": { 00:23:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG [...] 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG 00:23:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:23:45 DEBUG }, 00:23:45 DEBUG "value": [ 00:23:45 DEBUG 1713050625, 00:23:45 DEBUG "2100.6666666666665" 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG ] 00:23:45 DEBUG } 00:23:45 DEBUG } 00:23:45 INFO 00:23:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:23:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:23:45 INFO [loop_until]: OK (rc = 0) 00:23:45 DEBUG --- stdout --- 00:23:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11485m 8991Mi am-7d597cf7d6-8vr8f 10067m 8364Mi am-7d597cf7d6-9wz9h 14005m 9063Mi ds-cts-0 10365m 6842Mi ds-cts-1 10579m 7072Mi ds-cts-2 10434m 6950Mi ds-idrepo-0 12245m 4227Mi ds-idrepo-1 1804m 4282Mi ds-idrepo-2 1682m 4099Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 7m 959Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 1m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5737m 1615Mi 00:23:45 DEBUG --- stderr --- 00:23:45 DEBUG 00:23:46 INFO 00:23:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:23:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:23:46 INFO [loop_until]: OK (rc = 0) 00:23:46 DEBUG --- stdout --- 00:23:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 4801Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10505m 35% 8991Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11559m 38% 11046Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15562m 52% 2985Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1858m 6% 6272Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10218m 34% 10205Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2880Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11018m 36% 9786Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1805m 6% 5427Mi 4% gke-xlou-cdm-ds-0f601751-j488 10781m 36% 8435Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 66m 0% 2232Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14705m 49% 10592Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 12011m 40% 5605Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6040m 38% 5382Mi 9% 00:23:46 DEBUG --- stderr --- 00:23:46 DEBUG 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG { 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG "metric": { 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG [...] 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG 00:24:45 DEBUG { 00:24:45 DEBUG 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG 00:24:45 DEBUG "data": { 00:24:45 DEBUG "pod": "ds-cts-1" 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG }, 00:24:45 DEBUG "result": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG { 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "0.00014394444444444445" 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG ] 00:24:45 DEBUG }, 00:24:45 DEBUG } 00:24:45 DEBUG "value": [ 00:24:45 DEBUG ] 00:24:45 DEBUG 00:24:45 DEBUG } 00:24:45 DEBUG 00:24:45 DEBUG } 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "2088.8" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "2088.5333333333333" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 ERROR [http_cmd]: ERROR 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 ERROR --- status code --- 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG --- status code --- 00:24:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 ERROR --- http response --- 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "data": { 00:24:45 DEBUG "data": { 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG "result": [ 00:24:45 DEBUG "result": [ 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG "instance": "10.142.0.107:9100" 00:24:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG [...] 00:24:45 DEBUG [...] 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG "instance": "10.142.0.43:9100" 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG "0" 00:24:45 DEBUG "12572402688" 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050685" 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "2088.8" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "instance": "10.142.0.107:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "instance": "10.142.0.43:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "1794285568" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0.008" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG }, 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG [...] 00:24:45 DEBUG [...] 00:24:45 DEBUG { 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "data": { 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:24:45 DEBUG "pod": "ds-idrepo-0" 00:24:45 DEBUG "result": [ 00:24:45 DEBUG }, 00:24:45 DEBUG }, 00:24:45 DEBUG { 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "value": [ 00:24:45 DEBUG "metric": { 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "instance": "10.142.0.107:9100" 00:24:45 DEBUG "5.91965812073992" 00:24:45 DEBUG "0" 00:24:45 DEBUG }, 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG "value": [ 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG 00:24:45 DEBUG ] 00:24:45 DEBUG ] 00:24:45 DEBUG 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "instance": "10.142.0.43:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0.000333333333333267" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "2114.5333333333333" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "instance": "10.142.0.107:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "instance": "10.142.0.43:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "5210" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "ds-idrepo-1" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "ds-cts-1" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050685" 00:24:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050685" 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "379577.5" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "0" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO [http_cmd]: http status code OK 00:24:45 DEBUG --- status code --- 00:24:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:24:45 DEBUG --- http response --- 00:24:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:24:45 DEBUG ----- output ----- 00:24:45 DEBUG { 00:24:45 DEBUG "status": "success", 00:24:45 DEBUG "data": { 00:24:45 DEBUG "resultType": "vector", 00:24:45 DEBUG "result": [ 00:24:45 DEBUG { 00:24:45 DEBUG "metric": { 00:24:45 DEBUG "instance": "10.142.0.107:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG [...] 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG 00:24:45 DEBUG "instance": "10.142.0.43:9100" 00:24:45 DEBUG }, 00:24:45 DEBUG "value": [ 00:24:45 DEBUG 1713050685, 00:24:45 DEBUG "12099.93333333333" 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG ] 00:24:45 DEBUG } 00:24:45 DEBUG } 00:24:45 INFO 00:24:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:24:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:24:45 INFO [loop_until]: OK (rc = 0) 00:24:45 DEBUG --- stdout --- 00:24:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11369m 8993Mi am-7d597cf7d6-8vr8f 10115m 8366Mi am-7d597cf7d6-9wz9h 13654m 9079Mi ds-cts-0 10314m 6849Mi ds-cts-1 10403m 7193Mi ds-cts-2 10306m 6957Mi ds-idrepo-0 12113m 4228Mi ds-idrepo-1 1783m 4283Mi ds-idrepo-2 1746m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 960Mi idm-b85f9d978-rl76f 8m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 63Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5981m 1616Mi 00:24:45 DEBUG --- stderr --- 00:24:45 DEBUG 00:24:46 INFO 00:24:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:24:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:24:46 INFO [loop_until]: OK (rc = 0) 00:24:46 DEBUG --- stdout --- 00:24:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 4804Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10592m 35% 8994Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11568m 38% 11047Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15545m 52% 2984Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1912m 6% 6277Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10363m 34% 10211Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2881Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11076m 37% 9787Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1827m 6% 5428Mi 4% gke-xlou-cdm-ds-0f601751-j488 10675m 35% 8561Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2219Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14849m 49% 10608Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11868m 39% 5612Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6111m 38% 5385Mi 9% 00:24:46 DEBUG --- stderr --- 00:24:46 DEBUG 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 ERROR [http_cmd]: ERROR 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 ERROR --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 ERROR --- http response --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:25:45 DEBUG --- http response --- 00:25:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG { 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG { 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG { 00:25:45 DEBUG "data": { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG "data": { 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG "result": [ 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "result": [ 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG { 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "instance": "10.142.0.107:9100" 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG "metric": { 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 DEBUG }, 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "data": { 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 DEBUG }, 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG }, 00:25:45 DEBUG "result": [ 00:25:45 DEBUG 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "value": [ 00:25:45 DEBUG { 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "metric": { 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG }, 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "instance": "10.142.0.43:9100" 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG }, 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG [...] 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "8.595467672647946" 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG }, 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG "pod": "ds-cts-1" 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG ] 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0" 00:25:45 DEBUG "value": [ 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG "2098.5333333333333" 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG }, 00:25:45 DEBUG "2098.6" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG "4299.561650012811" 00:25:45 DEBUG ] 00:25:45 DEBUG { 00:25:45 DEBUG 00:25:45 DEBUG ] 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:25:45 DEBUG "0" 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG } 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG ] 00:25:45 DEBUG "0" 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "12572935168" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG "2098.6" 00:25:45 DEBUG "data": { 00:25:45 DEBUG }, 00:25:45 DEBUG ] 00:25:45 DEBUG { 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG "0" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG "value": [ 00:25:45 DEBUG } 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG } 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG "result": [ 00:25:45 DEBUG ] 00:25:45 DEBUG "0" 00:25:45 DEBUG } 00:25:45 DEBUG "result": [ 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG { 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG ] 00:25:45 DEBUG { 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG "metric": { 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG "metric": { 00:25:45 DEBUG } 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:25:45 DEBUG }, 00:25:45 DEBUG } 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG } 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG }, 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0" 00:25:45 DEBUG "2941.805339482449" 00:25:45 DEBUG ] 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "instance": "10.142.0.107:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "instance": "10.142.0.43:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0.0005666666666667671" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050745" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "1796296704" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "instance": "10.142.0.107:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "instance": "10.142.0.43:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0.008" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "5.903197468067109" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "instance": "10.142.0.107:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "instance": "10.142.0.43:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "6784.5" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "2117.7" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "ds-idrepo-1" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "ds-cts-1" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "378958.5" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050745" 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "pod": "ds-idrepo-0" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "0" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO [http_cmd]: http status code OK 00:25:45 DEBUG --- status code --- 00:25:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:25:45 DEBUG --- http response --- 00:25:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:25:45 DEBUG ----- output ----- 00:25:45 DEBUG { 00:25:45 DEBUG "status": "success", 00:25:45 DEBUG "data": { 00:25:45 DEBUG "resultType": "vector", 00:25:45 DEBUG "result": [ 00:25:45 DEBUG { 00:25:45 DEBUG "metric": { 00:25:45 DEBUG "instance": "10.142.0.107:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG [...] 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG 00:25:45 DEBUG "instance": "10.142.0.43:9100" 00:25:45 DEBUG }, 00:25:45 DEBUG "value": [ 00:25:45 DEBUG 1713050745, 00:25:45 DEBUG "12319.766666666665" 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG ] 00:25:45 DEBUG } 00:25:45 DEBUG } 00:25:45 INFO 00:25:45 INFO [loop_until]: kubectl --namespace=xlou top pods 00:25:45 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:25:45 INFO [loop_until]: OK (rc = 0) 00:25:45 DEBUG --- stdout --- 00:25:45 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11107m 8987Mi am-7d597cf7d6-8vr8f 9973m 8377Mi am-7d597cf7d6-9wz9h 13269m 9067Mi ds-cts-0 10266m 6834Mi ds-cts-1 10255m 7195Mi ds-cts-2 10401m 7015Mi ds-idrepo-0 11925m 4229Mi ds-idrepo-1 1820m 4284Mi ds-idrepo-2 1725m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 960Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5875m 1619Mi 00:25:45 DEBUG --- stderr --- 00:25:45 DEBUG 00:25:46 INFO 00:25:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:25:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:25:46 INFO [loop_until]: OK (rc = 0) 00:25:46 DEBUG --- stdout --- 00:25:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 4821Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10530m 35% 9094Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11625m 38% 11052Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15535m 52% 2987Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1903m 6% 6278Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10139m 33% 10195Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2883Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10625m 35% 9800Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1812m 6% 5425Mi 4% gke-xlou-cdm-ds-0f601751-j488 10537m 35% 8578Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2225Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14756m 49% 10592Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11933m 39% 5604Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 5815m 36% 5385Mi 9% 00:25:46 DEBUG --- stderr --- 00:25:46 DEBUG 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "2094.0333333333333" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 ERROR [http_cmd]: ERROR 00:26:45 ERROR --- status code --- 00:26:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:26:45 ERROR --- http response --- 00:26:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:26:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG --- status code --- 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG --- http response --- 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG ----- output ----- 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 DEBUG ----- output ----- 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG { 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG { 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG { 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG "data": { 00:26:45 DEBUG "data": { 00:26:45 DEBUG "data": { 00:26:45 DEBUG { 00:26:45 DEBUG "data": { 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:26:45 DEBUG "data": { 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG "data": { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "data": { 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "result": [ 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "data": { 00:26:45 DEBUG { 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "data": { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "instance": "10.142.0.107:9100" 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG { 00:26:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "metric": { 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "instance": "10.142.0.107:9100" 00:26:45 DEBUG "data": { 00:26:45 DEBUG }, 00:26:45 DEBUG }, 00:26:45 DEBUG }, 00:26:45 DEBUG "metric": { 00:26:45 DEBUG }, 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:26:45 DEBUG }, 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG }, 00:26:45 DEBUG "result": [ 00:26:45 DEBUG }, 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "instance": "10.142.0.107:9100" 00:26:45 DEBUG "value": [ 00:26:45 DEBUG { 00:26:45 DEBUG }, 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG { 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "result": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "value": [ 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG "metric": { 00:26:45 DEBUG 00:26:45 DEBUG { 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:26:45 DEBUG }, 00:26:45 DEBUG "metric": { 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG [...] 00:26:45 DEBUG [...] 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:26:45 DEBUG [...] 00:26:45 DEBUG [...] 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG "instance": "10.142.0.43:9100" 00:26:45 DEBUG "pod": "ds-idrepo-0" 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG { 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "ds-idrepo-0" 00:26:45 DEBUG 00:26:45 DEBUG "pod": "ds-cts-1" 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG }, 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG }, 00:26:45 DEBUG "instance": "10.142.0.43:9100" 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG }, 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0.00036666666666652263" 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG "instance": "10.142.0.43:9100" 00:26:45 DEBUG "data": { 00:26:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:26:45 DEBUG "pod": "ds-idrepo-0" 00:26:45 DEBUG "pod": "ds-idrepo-0" 00:26:45 DEBUG "value": [ 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG ] 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG }, 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG }, 00:26:45 DEBUG }, 00:26:45 DEBUG }, 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG 00:26:45 DEBUG "2094.0333333333333" 00:26:45 DEBUG "0" 00:26:45 DEBUG } 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "0" 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0.00019999999999999998" 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG }, 00:26:45 DEBUG 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG "2855129.094562857" 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG { 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG ] 00:26:45 DEBUG "6.406753991285601" 00:26:45 DEBUG ] 00:26:45 DEBUG "12573543424" 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "336739307.0377163" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0" 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "1794105344" 00:26:45 DEBUG "0" 00:26:45 DEBUG "0" 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG "0.00033333333333333365" 00:26:45 DEBUG ] 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG }, 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG }, 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG "value": [ 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG "value": [ 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG "2094.133333333333" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG [...] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG 00:26:45 DEBUG } 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "ds-idrepo-0" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0.008" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050805" 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "2114.4666666666667" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:26:45 DEBUG "instance": "10.142.0.107:9100" 00:26:45 DEBUG }, 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG "instance": "10.142.0.43:9100" 00:26:45 DEBUG }, 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0" 00:26:45 DEBUG "5290.7" 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "5.9075483010963" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050805" 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "ds-idrepo-1" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "ds-cts-1" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "0" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "instance": "10.142.0.107:9100" 00:26:45 DEBUG }, 00:26:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050805" 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "instance": "10.142.0.43:9100" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "12187.366666666665" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:45 INFO [http_cmd]: http status code OK 00:26:45 DEBUG --- status code --- 00:26:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:26:45 DEBUG --- http response --- 00:26:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:26:45 DEBUG ----- output ----- 00:26:45 DEBUG { 00:26:45 DEBUG "status": "success", 00:26:45 DEBUG "data": { 00:26:45 DEBUG "resultType": "vector", 00:26:45 DEBUG "result": [ 00:26:45 DEBUG { 00:26:45 DEBUG "metric": { 00:26:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG [...] 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG 00:26:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:26:45 DEBUG }, 00:26:45 DEBUG "value": [ 00:26:45 DEBUG 1713050805, 00:26:45 DEBUG "377925.5" 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG ] 00:26:45 DEBUG } 00:26:45 DEBUG } 00:26:46 INFO 00:26:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:26:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:26:46 INFO [loop_until]: OK (rc = 0) 00:26:46 DEBUG --- stdout --- 00:26:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11321m 8989Mi am-7d597cf7d6-8vr8f 10329m 8365Mi am-7d597cf7d6-9wz9h 13760m 9062Mi ds-cts-0 10036m 6894Mi ds-cts-1 10794m 7202Mi ds-cts-2 10397m 7036Mi ds-idrepo-0 12189m 4230Mi ds-idrepo-1 1842m 4284Mi ds-idrepo-2 1737m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 960Mi idm-b85f9d978-rl76f 4m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5822m 1620Mi 00:26:46 DEBUG --- stderr --- 00:26:46 DEBUG 00:26:46 INFO 00:26:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:26:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:26:46 INFO [loop_until]: OK (rc = 0) 00:26:46 DEBUG --- stdout --- 00:26:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4804Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10339m 34% 9097Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11624m 38% 11050Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15558m 52% 2987Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1903m 6% 6273Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10049m 33% 10240Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2890Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10978m 36% 9788Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1796m 6% 5434Mi 4% gke-xlou-cdm-ds-0f601751-j488 10749m 36% 8572Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2222Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14778m 49% 10599Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11982m 40% 5606Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6034m 37% 5381Mi 9% 00:26:46 DEBUG --- stderr --- 00:26:46 DEBUG 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050865" 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "data": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "2110.5333333333333" 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:27:45 ERROR [http_cmd]: ERROR 00:27:45 ERROR --- status code --- 00:27:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:27:45 ERROR --- http response --- 00:27:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:27:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050865" 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "data": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "instance": "10.142.0.107:9100" 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "instance": "10.142.0.43:9100" 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "0.0003333333333333262" 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- status code --- 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG { 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG ----- output ----- 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG --- status code --- 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050865" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050865" 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG { 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:27:45 DEBUG "data": { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "data": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "data": { 00:27:45 DEBUG "data": { 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "data": { 00:27:45 DEBUG "data": { 00:27:45 DEBUG "data": { 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG "data": { 00:27:45 DEBUG { 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "data": { 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "data": { 00:27:45 DEBUG "data": { 00:27:45 DEBUG "data": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG "data": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "data": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "result": [ 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG { 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "result": [ 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG }, 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "data": { 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:27:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG "data": { 00:27:45 DEBUG "instance": "10.142.0.107:9100" 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG }, 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG "data": { 00:27:45 DEBUG "instance": "10.142.0.107:9100" 00:27:45 DEBUG 00:27:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:27:45 DEBUG }, 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG }, 00:27:45 DEBUG }, 00:27:45 DEBUG "result": [ 00:27:45 DEBUG { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG }, 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG }, 00:27:45 DEBUG }, 00:27:45 DEBUG }, 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "value": [ 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:27:45 DEBUG "value": [ 00:27:45 DEBUG { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "result": [ 00:27:45 DEBUG ----- output ----- 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG "value": [ 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "data": { 00:27:45 DEBUG { 00:27:45 DEBUG { 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG { 00:27:45 DEBUG 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG "data": { 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG [...] 00:27:45 DEBUG "data": { 00:27:45 DEBUG [...] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG { 00:27:45 DEBUG "metric": { 00:27:45 DEBUG }, 00:27:45 DEBUG [...] 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG [...] 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG "pod": "ds-idrepo-0" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "result": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "result": [ 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG { 00:27:45 DEBUG 00:27:45 DEBUG "pod": "ds-idrepo-2" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG { 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG { 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG "data": { 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG "pod": "ds-idrepo-0" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG "pod": "ds-idrepo-2" 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "ds-idrepo-2" 00:27:45 DEBUG [...] 00:27:45 DEBUG "metric": { 00:27:45 DEBUG 00:27:45 DEBUG "instance": "10.142.0.43:9100" 00:27:45 DEBUG 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "pod": "ds-cts-1" 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050865" 00:27:45 DEBUG "pod": "ds-idrepo-0" 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG "0" 00:27:45 DEBUG }, 00:27:45 DEBUG }, 00:27:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:27:45 DEBUG }, 00:27:45 DEBUG }, 00:27:45 DEBUG "instance": "10.142.0.107:9100" 00:27:45 DEBUG }, 00:27:45 DEBUG }, 00:27:45 DEBUG [...] 00:27:45 DEBUG "result": [ 00:27:45 DEBUG 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG 00:27:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:27:45 DEBUG ] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG { 00:27:45 DEBUG "pod": "ds-idrepo-0" 00:27:45 DEBUG }, 00:27:45 DEBUG "31382.998258667085" 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "0.00014394444444444445" 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG "0" 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG 00:27:45 DEBUG "1.7118492005698906" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "pod": "ds-idrepo-0" 00:27:45 DEBUG "value": [ 00:27:45 DEBUG ] 00:27:45 DEBUG "2110.4333333333334" 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG "instance": "10.142.0.43:9100" 00:27:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050865" 00:27:45 DEBUG ] 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "5446.63606142156" 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:27:45 DEBUG }, 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG }, 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG ] 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG [...] 00:27:45 DEBUG ] 00:27:45 DEBUG "12574166016" 00:27:45 DEBUG [...] 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG "value": [ 00:27:45 DEBUG }, 00:27:45 DEBUG "0" 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG "0" 00:27:45 DEBUG } 00:27:45 DEBUG }, 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "0" 00:27:45 DEBUG "1795784704" 00:27:45 DEBUG } 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG ] 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "0.000500000000000019" 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG } 00:27:45 DEBUG 00:27:45 DEBUG } 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG ] 00:27:45 DEBUG "pod": "ds-idrepo-2" 00:27:45 DEBUG ] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "instance": "10.142.0.43:9100" 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG [...] 00:27:45 DEBUG ] 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG "0.008" 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:27:45 DEBUG } 00:27:45 DEBUG }, 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG }, 00:27:45 DEBUG "0" 00:27:45 DEBUG 00:27:45 DEBUG } 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG "2110.9" 00:27:45 DEBUG }, 00:27:45 DEBUG } 00:27:45 DEBUG "value": [ 00:27:45 DEBUG } 00:27:45 DEBUG "2088.366666666667" 00:27:45 DEBUG ] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG } 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 INFO [http_cmd]: http status code OK 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "0" 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG "4903.633333333333" 00:27:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:27:45 DEBUG } 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:27:45 DEBUG --- status code --- 00:27:45 DEBUG ] 00:27:45 DEBUG "377406.5" 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG }, 00:27:45 DEBUG } 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG "value": [ 00:27:45 DEBUG { 00:27:45 DEBUG --- http response --- 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG } 00:27:45 DEBUG "data": { 00:27:45 DEBUG ----- output ----- 00:27:45 DEBUG } 00:27:45 DEBUG "5.897443506221061" 00:27:45 DEBUG } 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG { 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG "result": [ 00:27:45 DEBUG "status": "success", 00:27:45 DEBUG } 00:27:45 DEBUG { 00:27:45 DEBUG "data": { 00:27:45 DEBUG ] 00:27:45 DEBUG "metric": { 00:27:45 DEBUG "resultType": "vector", 00:27:45 DEBUG } 00:27:45 DEBUG "pod": "ds-idrepo-1" 00:27:45 DEBUG "result": [ 00:27:45 DEBUG } 00:27:45 DEBUG }, 00:27:45 DEBUG { 00:27:45 DEBUG "value": [ 00:27:45 DEBUG "metric": { 00:27:45 DEBUG 00:27:45 DEBUG "instance": "10.142.0.107:9100" 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG [...] 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG 00:27:45 DEBUG [...] 00:27:45 DEBUG "pod": "ds-cts-1" 00:27:45 DEBUG 00:27:45 DEBUG }, 00:27:45 DEBUG 00:27:45 DEBUG "value": [ 00:27:45 DEBUG 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG "instance": "10.142.0.43:9100" 00:27:45 DEBUG "0" 00:27:45 DEBUG }, 00:27:45 DEBUG ] 00:27:45 DEBUG "value": [ 00:27:45 DEBUG } 00:27:45 DEBUG 1713050865, 00:27:45 DEBUG ] 00:27:45 DEBUG "12144.3" 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:45 DEBUG ] 00:27:45 DEBUG } 00:27:45 DEBUG } 00:27:46 INFO 00:27:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:27:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:27:46 INFO [loop_until]: OK (rc = 0) 00:27:46 DEBUG --- stdout --- 00:27:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11507m 9004Mi am-7d597cf7d6-8vr8f 10036m 8366Mi am-7d597cf7d6-9wz9h 13691m 9065Mi ds-cts-0 9949m 6996Mi ds-cts-1 10315m 7399Mi ds-cts-2 10676m 7205Mi ds-idrepo-0 12250m 4232Mi ds-idrepo-1 1809m 4285Mi ds-idrepo-2 1626m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 960Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5812m 1622Mi 00:27:46 DEBUG --- stderr --- 00:27:46 DEBUG 00:27:46 INFO 00:27:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:27:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:27:46 INFO [loop_until]: OK (rc = 0) 00:27:46 DEBUG --- stdout --- 00:27:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4796Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10769m 36% 9254Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11532m 38% 11061Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15133m 50% 2978Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1912m 6% 6273Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 9883m 33% 10368Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2881Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11084m 37% 9792Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1749m 5% 5431Mi 4% gke-xlou-cdm-ds-0f601751-j488 10712m 35% 8777Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 59m 0% 2229Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14893m 49% 10591Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11987m 40% 5619Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6040m 38% 5383Mi 9% 00:27:46 DEBUG --- stderr --- 00:27:46 DEBUG 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "data": { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "2106.366666666667" 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050925" 00:28:45 ERROR [http_cmd]: ERROR 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 ERROR --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 ERROR --- http response --- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:28:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:28:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050925" 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "data": { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "instance": "10.142.0.107:9100" 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "instance": "10.142.0.43:9100" 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "0.0003333333333333262" 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050925" 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050925" 00:28:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050925" 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- status code --- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- status code --- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- status code --- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG --- status code --- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- http response --- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG { 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG ----- output ----- 00:28:45 INFO [http_cmd]: http status code OK 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG --- status code --- 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "data": { 00:28:45 DEBUG { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG { 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG { 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG { 00:28:45 DEBUG "data": { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG { 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG "data": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "data": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "data": { 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG { 00:28:45 DEBUG "data": { 00:28:45 DEBUG "data": { 00:28:45 DEBUG --- http response --- 00:28:45 DEBUG "data": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "data": { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "data": { 00:28:45 DEBUG "data": { 00:28:45 DEBUG "data": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "data": { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "data": { 00:28:45 DEBUG { 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "data": { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "data": { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "data": { 00:28:45 DEBUG { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG { 00:28:45 DEBUG "data": { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "result": [ 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "result": [ 00:28:45 DEBUG ----- output ----- 00:28:45 DEBUG "result": [ 00:28:45 DEBUG { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "data": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG { 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "data": { 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "data": { 00:28:45 DEBUG { 00:28:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "status": "success", 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG { 00:28:45 DEBUG }, 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG "metric": { 00:28:45 DEBUG }, 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG "data": { 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG "data": { 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG }, 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:28:45 DEBUG "instance": "10.142.0.107:9100" 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "instance": "10.142.0.107:9100" 00:28:45 DEBUG }, 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG { 00:28:45 DEBUG "metric": { 00:28:45 DEBUG { 00:28:45 DEBUG }, 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:28:45 DEBUG }, 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG }, 00:28:45 DEBUG "resultType": "vector", 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG }, 00:28:45 DEBUG }, 00:28:45 DEBUG }, 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG }, 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "value": [ 00:28:45 DEBUG { 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "result": [ 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "value": [ 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "instance": "10.142.0.107:9100" 00:28:45 DEBUG }, 00:28:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:28:45 DEBUG 00:28:45 DEBUG "metric": { 00:28:45 DEBUG "pod": "ds-idrepo-1" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG { 00:28:45 DEBUG 00:28:45 DEBUG { 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "metric": { 00:28:45 DEBUG 00:28:45 DEBUG "metric": { 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "instance": "10.142.0.107:9100" 00:28:45 DEBUG 00:28:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG [...] 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG "instance": "10.142.0.43:9100" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG [...] 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG "instance": "10.142.0.43:9100" 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG }, 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG [...] 00:28:45 DEBUG }, 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG "pod": "ds-cts-1" 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG "pod": "ds-cts-1" 00:28:45 DEBUG 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG }, 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "378343.5" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "5697845.921138574" 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG [...] 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG }, 00:28:45 DEBUG "0" 00:28:45 DEBUG "12574858240" 00:28:45 DEBUG "value": [ 00:28:45 DEBUG }, 00:28:45 DEBUG "0.00040000000000001514" 00:28:45 DEBUG "instance": "10.142.0.43:9100" 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "pod": "ds-idrepo-0" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG ] 00:28:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:28:45 DEBUG ] 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG 00:28:45 DEBUG "value": [ 00:28:45 DEBUG "instance": "10.142.0.43:9100" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "2106.3" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG }, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "value": [ 00:28:45 DEBUG ] 00:28:45 DEBUG }, 00:28:45 DEBUG 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG }, 00:28:45 DEBUG "0" 00:28:45 DEBUG } 00:28:45 DEBUG }, 00:28:45 DEBUG } 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "0" 00:28:45 DEBUG 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG }, 00:28:45 DEBUG "0" 00:28:45 DEBUG ] 00:28:45 DEBUG "0" 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG "0" 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG "value": [ 00:28:45 DEBUG 00:28:45 DEBUG "0.0085" 00:28:45 DEBUG "value": [ 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG "value": [ 00:28:45 DEBUG ] 00:28:45 DEBUG "0" 00:28:45 DEBUG ] 00:28:45 DEBUG 00:28:45 DEBUG "0.0001" 00:28:45 DEBUG "value": [ 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG "2106.1666666666665" 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG "0" 00:28:45 DEBUG ] 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG 00:28:45 DEBUG ] 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:28:45 DEBUG ] 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG "0" 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG "7007.2" 00:28:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:28:45 DEBUG } 00:28:45 DEBUG "0" 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG "2098.8333333333335" 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG }, 00:28:45 DEBUG } 00:28:45 DEBUG "12198.33333333333" 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG }, 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG "value": [ 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG "value": [ 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG ] 00:28:45 DEBUG 1713050925, 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG "5.928701376026378" 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG "1802493952" 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:45 DEBUG ] 00:28:45 DEBUG } 00:28:45 DEBUG } 00:28:46 INFO 00:28:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:28:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:28:46 INFO [loop_until]: OK (rc = 0) 00:28:46 DEBUG --- stdout --- 00:28:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10949m 8991Mi am-7d597cf7d6-8vr8f 10456m 8366Mi am-7d597cf7d6-9wz9h 13802m 9065Mi ds-cts-0 10117m 7017Mi ds-cts-1 10456m 7406Mi ds-cts-2 10659m 7208Mi ds-idrepo-0 12088m 4241Mi ds-idrepo-1 1809m 4285Mi ds-idrepo-2 1720m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 8m 961Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 10m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5924m 1623Mi 00:28:46 DEBUG --- stderr --- 00:28:46 DEBUG 00:28:46 INFO 00:28:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:28:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:28:46 INFO [loop_until]: OK (rc = 0) 00:28:46 DEBUG --- stdout --- 00:28:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4802Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10527m 35% 9255Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11449m 38% 11041Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15646m 52% 2974Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1881m 6% 6276Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10421m 34% 10397Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2881Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11156m 37% 9792Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1780m 5% 5426Mi 4% gke-xlou-cdm-ds-0f601751-j488 10634m 35% 8786Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2226Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14851m 49% 10599Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11924m 39% 5609Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6070m 38% 5387Mi 9% 00:28:46 DEBUG --- stderr --- 00:28:46 DEBUG 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "2103.0333333333333" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "instance": "10.142.0.107:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "instance": "10.142.0.43:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "0" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 ERROR [http_cmd]: ERROR 00:29:45 ERROR --- status code --- 00:29:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:29:45 ERROR --- http response --- 00:29:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:29:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "ds-cts-1" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG "0" 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG ] 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "instance": "10.142.0.107:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "instance": "10.142.0.43:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "5215.566666666667" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "instance": "10.142.0.107:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "instance": "10.142.0.43:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "0.00036666666666675953" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "2096.866666666667" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "0" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "ds-cts-1" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "0.00023333333333333333" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713050985" 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713050985" 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- status code --- 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG --- http response --- 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG --- http response --- 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG { 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:29:45 INFO [http_cmd]: http status code OK 00:29:45 DEBUG { 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG { 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG --- status code --- 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG { 00:29:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:29:45 DEBUG "data": { 00:29:45 DEBUG { 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "data": { 00:29:45 DEBUG "data": { 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG "data": { 00:29:45 DEBUG "data": { 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG "data": { 00:29:45 DEBUG "data": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG --- http response --- 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "data": { 00:29:45 DEBUG "result": [ 00:29:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:29:45 DEBUG "data": { 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG ----- output ----- 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG "data": { 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG "data": { 00:29:45 DEBUG { 00:29:45 DEBUG { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG { 00:29:45 DEBUG "result": [ 00:29:45 DEBUG { 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG }, 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "instance": "10.142.0.107:9100" 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:29:45 DEBUG }, 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:29:45 DEBUG "result": [ 00:29:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG }, 00:29:45 DEBUG { 00:29:45 DEBUG }, 00:29:45 DEBUG "status": "success", 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG { 00:29:45 DEBUG }, 00:29:45 DEBUG }, 00:29:45 DEBUG { 00:29:45 DEBUG }, 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "data": { 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "metric": { 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG 00:29:45 DEBUG "resultType": "vector", 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "ds-idrepo-1" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "instance": "10.142.0.107:9100" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG "result": [ 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG { 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG "metric": { 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG [...] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "378614.5" 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "pod": "ds-idrepo-0" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "pod": "ds-idrepo-0" 00:29:45 DEBUG 00:29:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG [...] 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG }, 00:29:45 DEBUG "pod": "ds-cts-2" 00:29:45 DEBUG "instance": "10.142.0.43:9100" 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "pod": "ds-idrepo-0" 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG }, 00:29:45 DEBUG [...] 00:29:45 DEBUG ] 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG 00:29:45 DEBUG "2098.7666666666664" 00:29:45 DEBUG ] 00:29:45 DEBUG }, 00:29:45 DEBUG }, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "1802715136" 00:29:45 DEBUG 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 00:29:45 DEBUG } 00:29:45 DEBUG "0" 00:29:45 DEBUG 00:29:45 DEBUG "value": [ 00:29:45 DEBUG } 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "pod": "ds-idrepo-0" 00:29:45 DEBUG [...] 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG ] 00:29:45 DEBUG [...] 00:29:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:29:45 DEBUG "0" 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "0" 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG "0.009" 00:29:45 DEBUG }, 00:29:45 DEBUG 00:29:45 DEBUG "2098.6666666666665" 00:29:45 DEBUG } 00:29:45 DEBUG 00:29:45 DEBUG }, 00:29:45 DEBUG ] 00:29:45 DEBUG "0" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG 00:29:45 DEBUG "instance": "10.142.0.43:9100" 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG "0" 00:29:45 DEBUG "12575441920" 00:29:45 DEBUG ] 00:29:45 DEBUG "value": [ 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG 00:29:45 DEBUG 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG }, 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG 00:29:45 DEBUG "0" 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "pod": "ds-cts-2" 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG "pod": "ds-idrepo-0" 00:29:45 DEBUG } 00:29:45 DEBUG "value": [ 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG }, 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG }, 00:29:45 DEBUG } 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG "0" 00:29:45 DEBUG "value": [ 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG "value": [ 00:29:45 DEBUG "11841.466666666664" 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG }, 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG "value": [ 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG "0" 00:29:45 DEBUG "0" 00:29:45 DEBUG } 00:29:45 DEBUG 1713050985, 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG "5.913735808518697" 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG ] 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:45 DEBUG } 00:29:46 INFO 00:29:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:29:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:29:46 INFO [loop_until]: OK (rc = 0) 00:29:46 DEBUG --- stdout --- 00:29:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11187m 8989Mi am-7d597cf7d6-8vr8f 10215m 8366Mi am-7d597cf7d6-9wz9h 14103m 9065Mi ds-cts-0 10033m 7012Mi ds-cts-1 10458m 7429Mi ds-cts-2 10505m 7213Mi ds-idrepo-0 11971m 4249Mi ds-idrepo-1 1865m 4286Mi ds-idrepo-2 1797m 4101Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 7m 961Mi idm-b85f9d978-rl76f 4m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5819m 1624Mi 00:29:46 DEBUG --- stderr --- 00:29:46 DEBUG 00:29:46 INFO 00:29:46 INFO [loop_until]: kubectl --namespace=xlou top node 00:29:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:29:46 INFO [loop_until]: OK (rc = 0) 00:29:46 DEBUG --- stdout --- 00:29:46 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 4806Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10523m 35% 9257Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11587m 38% 11046Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15569m 52% 2978Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1898m 6% 6274Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10162m 34% 10383Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 77m 0% 2866Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11230m 37% 9794Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1802m 6% 5430Mi 4% gke-xlou-cdm-ds-0f601751-j488 10572m 35% 8790Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2220Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14840m 49% 10588Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11957m 40% 5624Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 5905m 37% 5386Mi 9% 00:29:46 DEBUG --- stderr --- 00:29:46 DEBUG 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "2101.4" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 ERROR [http_cmd]: ERROR 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 ERROR --- status code --- 00:30:45 DEBUG ----- output ----- 00:30:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:30:45 DEBUG { 00:30:45 ERROR --- http response --- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "instance": "10.142.0.107:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "instance": "10.142.0.43:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "0.00033333333333333365" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:30:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "instance": "10.142.0.107:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "instance": "10.142.0.43:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "6894.233333333333" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "ds-cts-1" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "0" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "instance": "10.142.0.107:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "instance": "10.142.0.43:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "0.00040000000000001514" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "2046" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG }, 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG 00:30:45 DEBUG "pod": "ds-cts-1" 00:30:45 DEBUG }, 00:30:45 DEBUG "pod": "ds-idrepo-0" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "0" 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "0.00011326388888888889" 00:30:45 DEBUG ] 00:30:45 DEBUG "0" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "0" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "instance": "10.142.0.107:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "instance": "10.142.0.43:9100" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "12575992832" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "101900.90714489421" 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051045" 00:30:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051045" 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- status code --- 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG { 00:30:45 INFO [http_cmd]: http status code OK 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG --- status code --- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG { 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG { 00:30:45 DEBUG { 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG { 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG "data": { 00:30:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:30:45 DEBUG { 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG --- http response --- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "data": { 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:30:45 DEBUG "data": { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "data": { 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "data": { 00:30:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:30:45 DEBUG { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG { 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG ----- output ----- 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG { 00:30:45 DEBUG { 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG { 00:30:45 DEBUG "status": "success", 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "data": { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG }, 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "data": { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "metric": { 00:30:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG { 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG "resultType": "vector", 00:30:45 DEBUG { 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG }, 00:30:45 DEBUG { 00:30:45 DEBUG "result": [ 00:30:45 DEBUG }, 00:30:45 DEBUG }, 00:30:45 DEBUG }, 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "metric": { 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG "result": [ 00:30:45 DEBUG "metric": { 00:30:45 DEBUG }, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "metric": { 00:30:45 DEBUG { 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "value": [ 00:30:45 DEBUG { 00:30:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG { 00:30:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:30:45 DEBUG "metric": { 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "metric": { 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "metric": { 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG "pod": "ds-idrepo-1" 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "instance": "10.142.0.107:9100" 00:30:45 DEBUG "value": [ 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG [...] 00:30:45 DEBUG [...] 00:30:45 DEBUG [...] 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:30:45 DEBUG 00:30:45 DEBUG [...] 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG "pod": "ds-idrepo-0" 00:30:45 DEBUG "pod": "ds-idrepo-0" 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "pod": "ds-idrepo-0" 00:30:45 DEBUG [...] 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG }, 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG }, 00:30:45 DEBUG 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG 00:30:45 DEBUG "379848" 00:30:45 DEBUG "value": [ 00:30:45 DEBUG 00:30:45 DEBUG }, 00:30:45 DEBUG }, 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:30:45 DEBUG 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:30:45 DEBUG "0" 00:30:45 DEBUG 00:30:45 DEBUG ] 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "value": [ 00:30:45 DEBUG "0" 00:30:45 DEBUG }, 00:30:45 DEBUG "pod": "ds-idrepo-0" 00:30:45 DEBUG "2101.5666666666666" 00:30:45 DEBUG "0" 00:30:45 DEBUG }, 00:30:45 DEBUG ] 00:30:45 DEBUG "instance": "10.142.0.43:9100" 00:30:45 DEBUG } 00:30:45 DEBUG "0" 00:30:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG ] 00:30:45 DEBUG "value": [ 00:30:45 DEBUG }, 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG "value": [ 00:30:45 DEBUG } 00:30:45 DEBUG }, 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG }, 00:30:45 DEBUG "0.009" 00:30:45 DEBUG "11.767474341352518" 00:30:45 DEBUG } 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG "value": [ 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG ] 00:30:45 DEBUG "value": [ 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG "value": [ 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG "2101.7" 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG "1803079680" 00:30:45 DEBUG } 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG 1713051045, 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG "0" 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG "12020.766666666665" 00:30:45 DEBUG } 00:30:45 DEBUG "5.938135033262016" 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG ] 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:45 DEBUG } 00:30:46 INFO 00:30:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:30:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:30:46 INFO [loop_until]: OK (rc = 0) 00:30:46 DEBUG --- stdout --- 00:30:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11098m 8986Mi am-7d597cf7d6-8vr8f 10007m 8371Mi am-7d597cf7d6-9wz9h 13559m 9072Mi ds-cts-0 10041m 7036Mi ds-cts-1 10054m 7416Mi ds-cts-2 10384m 7222Mi ds-idrepo-0 11928m 4249Mi ds-idrepo-1 1727m 4285Mi ds-idrepo-2 1706m 4101Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 961Mi idm-b85f9d978-rl76f 6m 967Mi lodemon-64cbdcc7b4-vt4kk 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5795m 1626Mi 00:30:46 DEBUG --- stderr --- 00:30:46 DEBUG 00:30:47 INFO 00:30:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:30:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:30:47 INFO [loop_until]: OK (rc = 0) 00:30:47 DEBUG --- stdout --- 00:30:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4818Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10532m 35% 9270Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11687m 39% 11045Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15641m 52% 2989Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1888m 6% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10315m 34% 10396Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2881Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11084m 37% 9796Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1763m 5% 5431Mi 4% gke-xlou-cdm-ds-0f601751-j488 10473m 35% 8799Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2207Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14915m 49% 10582Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11968m 40% 5627Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6052m 38% 5386Mi 9% 00:30:47 DEBUG --- stderr --- 00:30:47 DEBUG 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051105" 00:31:45 ERROR [http_cmd]: ERROR 00:31:45 ERROR --- status code --- 00:31:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:31:45 ERROR --- http response --- 00:31:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:31:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "instance": "10.142.0.107:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "instance": "10.142.0.43:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0.0003333333333333327" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "2100.5" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "2106.8333333333335" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "instance": "10.142.0.107:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "instance": "10.142.0.43:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "5277.633333333333" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "ds-cts-1" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG "3062868.6218330483" 00:31:45 DEBUG { 00:31:45 DEBUG ] 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG } 00:31:45 DEBUG "data": { 00:31:45 DEBUG ] 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG } 00:31:45 DEBUG "result": [ 00:31:45 DEBUG } 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "instance": "10.142.0.107:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "instance": "10.142.0.43:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0.00043333333333350765" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG { 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:31:45 DEBUG "data": { 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG }, 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "value": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG [...] 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG "pod": "ds-idrepo-0" 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG }, 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG 00:31:45 DEBUG "pod": "ds-cts-1" 00:31:45 DEBUG "0" 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG }, 00:31:45 DEBUG ] 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG } 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG ] 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0.0001666666666666667" 00:31:45 DEBUG } 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "instance": "10.142.0.107:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "instance": "10.142.0.43:9100" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "12576617472" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "ds-idrepo-0" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG { 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "ds-idrepo-0" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "75424.52435373272" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "2101.7" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051105" 00:31:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051105" 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "378508" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "pod": "ds-idrepo-0" 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:31:45 DEBUG ----- output ----- 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG { 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 INFO [http_cmd]: http status code OK 00:31:45 DEBUG "data": { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "data": { 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG --- status code --- 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "data": { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG "result": [ 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "result": [ 00:31:45 DEBUG --- http response --- 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG { 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG "result": [ 00:31:45 DEBUG { 00:31:45 DEBUG { 00:31:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:31:45 DEBUG "metric": { 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG { 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG ----- output ----- 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "instance": "10.142.0.107:9100" 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:31:45 DEBUG "data": { 00:31:45 DEBUG { 00:31:45 DEBUG }, 00:31:45 DEBUG "pod": "ds-idrepo-1" 00:31:45 DEBUG }, 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG }, 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG "status": "success", 00:31:45 DEBUG "value": [ 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG "result": [ 00:31:45 DEBUG "data": { 00:31:45 DEBUG 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG { 00:31:45 DEBUG "resultType": "vector", 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "metric": { 00:31:45 DEBUG "result": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:31:45 DEBUG { 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG }, 00:31:45 DEBUG "metric": { 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG "value": [ 00:31:45 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG }, 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG "value": [ 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG 00:31:45 DEBUG "instance": "10.142.0.43:9100" 00:31:45 DEBUG 00:31:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG }, 00:31:45 DEBUG "pod": "ds-idrepo-0" 00:31:45 DEBUG }, 00:31:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:31:45 DEBUG }, 00:31:45 DEBUG [...] 00:31:45 DEBUG 00:31:45 DEBUG "value": [ 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG }, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "value": [ 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG 00:31:45 DEBUG [...] 00:31:45 DEBUG "0.009" 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "12296.133333333331" 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG "11.324291053665137" 00:31:45 DEBUG 00:31:45 DEBUG 00:31:45 DEBUG ] 00:31:45 DEBUG "0" 00:31:45 DEBUG ] 00:31:45 DEBUG "2102.1666666666665" 00:31:45 DEBUG ] 00:31:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:31:45 DEBUG 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG }, 00:31:45 DEBUG 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG "value": [ 00:31:45 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG }, 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG "1801555968" 00:31:45 DEBUG "value": [ 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG 1713051105, 00:31:45 DEBUG } 00:31:45 DEBUG "5.964232669278637" 00:31:45 DEBUG ] 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:45 DEBUG ] 00:31:45 DEBUG } 00:31:45 DEBUG } 00:31:46 INFO 00:31:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:31:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:31:46 INFO [loop_until]: OK (rc = 0) 00:31:46 DEBUG --- stdout --- 00:31:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11358m 8988Mi am-7d597cf7d6-8vr8f 10115m 8370Mi am-7d597cf7d6-9wz9h 13722m 9063Mi ds-cts-0 10107m 7025Mi ds-cts-1 10419m 7417Mi ds-cts-2 10605m 7208Mi ds-idrepo-0 11955m 4250Mi ds-idrepo-1 1804m 4286Mi ds-idrepo-2 1671m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 961Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5988m 1627Mi 00:31:46 DEBUG --- stderr --- 00:31:46 DEBUG 00:31:47 INFO 00:31:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:31:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:31:47 INFO [loop_until]: OK (rc = 0) 00:31:47 DEBUG --- stdout --- 00:31:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4815Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10568m 35% 9257Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11655m 39% 11040Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15050m 50% 2990Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1937m 6% 6274Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10264m 34% 10417Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 2885Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11139m 37% 9794Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1709m 5% 5437Mi 4% gke-xlou-cdm-ds-0f601751-j488 10478m 35% 8799Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2212Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14869m 49% 10599Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11957m 40% 5633Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6044m 38% 5381Mi 9% 00:31:47 DEBUG --- stderr --- 00:31:47 DEBUG 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "2111.366666666667" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 ERROR [http_cmd]: ERROR 00:32:45 ERROR --- status code --- 00:32:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:32:45 ERROR --- http response --- 00:32:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:32:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "instance": "10.142.0.107:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "instance": "10.142.0.43:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "instance": "10.142.0.107:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "instance": "10.142.0.43:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "5249.566666666667" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "2098.7666666666664" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "instance": "10.142.0.107:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "instance": "10.142.0.43:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0.00040000000000001514" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "ds-idrepo-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "ds-cts-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "ds-idrepo-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "ds-idrepo-0" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "ds-idrepo-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "ds-cts-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0.00006666666666666667" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "instance": "10.142.0.107:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "instance": "10.142.0.43:9100" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "12577192960" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "ds-idrepo-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "ds-idrepo-0" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "ds-idrepo-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "ds-idrepo-0" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "2111.2666666666664" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "300068.77409287886" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "2111.2" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 INFO [http_cmd]: http status code OK 00:32:45 DEBUG "378846.5" 00:32:45 DEBUG --- status code --- 00:32:45 DEBUG ] 00:32:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:45 DEBUG } 00:32:45 DEBUG --- http response --- 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:32:45 DEBUG ----- output ----- 00:32:45 DEBUG { 00:32:45 DEBUG "status": "success", 00:32:45 DEBUG "data": { 00:32:45 DEBUG "resultType": "vector", 00:32:45 DEBUG "result": [ 00:32:45 DEBUG { 00:32:45 DEBUG "metric": { 00:32:45 DEBUG "pod": "ds-idrepo-1" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG [...] 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG 00:32:45 DEBUG "pod": "ds-idrepo-0" 00:32:45 DEBUG }, 00:32:45 DEBUG "value": [ 00:32:45 DEBUG 1713051165, 00:32:45 DEBUG "0" 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG ] 00:32:45 DEBUG } 00:32:45 DEBUG } 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051165" 00:32:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051166" 00:32:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051166" 00:32:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051166" 00:32:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051166" 00:32:46 INFO [http_cmd]: http status code OK 00:32:46 DEBUG --- status code --- 00:32:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:46 DEBUG --- http response --- 00:32:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:46 DEBUG ----- output ----- 00:32:46 DEBUG { 00:32:46 DEBUG "status": "success", 00:32:46 DEBUG "data": { 00:32:46 DEBUG "resultType": "vector", 00:32:46 DEBUG "result": [ 00:32:46 DEBUG { 00:32:46 DEBUG "metric": { 00:32:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG [...] 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 1713051165, 00:32:46 DEBUG "0.009" 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 INFO [http_cmd]: http status code OK 00:32:46 DEBUG --- status code --- 00:32:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:46 DEBUG --- http response --- 00:32:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:32:46 DEBUG ----- output ----- 00:32:46 DEBUG { 00:32:46 DEBUG "status": "success", 00:32:46 DEBUG "data": { 00:32:46 DEBUG "resultType": "vector", 00:32:46 DEBUG "result": [ 00:32:46 DEBUG { 00:32:46 DEBUG "metric": { 00:32:46 DEBUG "pod": "ds-idrepo-1" 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG [...] 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG "pod": "ds-idrepo-0" 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 1713051166, 00:32:46 DEBUG "0" 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 INFO [http_cmd]: http status code OK 00:32:46 INFO [http_cmd]: http status code OK 00:32:46 DEBUG --- status code --- 00:32:46 DEBUG --- status code --- 00:32:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:46 DEBUG --- http response --- 00:32:46 DEBUG --- http response --- 00:32:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:46 DEBUG ----- output ----- 00:32:46 DEBUG { 00:32:46 DEBUG "status": "success", 00:32:46 DEBUG "data": { 00:32:46 DEBUG "resultType": "vector", 00:32:46 DEBUG "result": [ 00:32:46 DEBUG { 00:32:46 DEBUG "metric": { 00:32:46 DEBUG "instance": "10.142.0.107:9100" 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG [...] 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG "instance": "10.142.0.43:9100" 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 1713051166, 00:32:46 DEBUG "12193.633333333331" 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 INFO [http_cmd]: http status code OK 00:32:46 INFO [http_cmd]: http status code OK 00:32:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:32:46 DEBUG --- status code --- 00:32:46 DEBUG --- status code --- 00:32:46 DEBUG ----- output ----- 00:32:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:32:46 DEBUG { 00:32:46 DEBUG --- http response --- 00:32:46 DEBUG --- http response --- 00:32:46 DEBUG "status": "success", 00:32:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:46 DEBUG "data": { 00:32:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:32:46 DEBUG ----- output ----- 00:32:46 DEBUG "resultType": "vector", 00:32:46 DEBUG ----- output ----- 00:32:46 DEBUG { 00:32:46 DEBUG "result": [ 00:32:46 DEBUG { 00:32:46 DEBUG "status": "success", 00:32:46 DEBUG { 00:32:46 DEBUG "status": "success", 00:32:46 DEBUG "data": { 00:32:46 DEBUG "metric": { 00:32:46 DEBUG "data": { 00:32:46 DEBUG "resultType": "vector", 00:32:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:32:46 DEBUG "resultType": "vector", 00:32:46 DEBUG "result": [ 00:32:46 DEBUG }, 00:32:46 DEBUG "result": [ 00:32:46 DEBUG { 00:32:46 DEBUG "value": [ 00:32:46 DEBUG { 00:32:46 DEBUG "metric": { 00:32:46 DEBUG 00:32:46 DEBUG "metric": { 00:32:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:32:46 DEBUG 00:32:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:32:46 DEBUG }, 00:32:46 DEBUG 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG [...] 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG [...] 00:32:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:32:46 DEBUG [...] 00:32:46 DEBUG 00:32:46 DEBUG }, 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 00:32:46 DEBUG 00:32:46 DEBUG 1713051165, 00:32:46 DEBUG 00:32:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:32:46 DEBUG "11.807053627401558" 00:32:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:32:46 DEBUG }, 00:32:46 DEBUG ] 00:32:46 DEBUG }, 00:32:46 DEBUG "value": [ 00:32:46 DEBUG } 00:32:46 DEBUG "value": [ 00:32:46 DEBUG 1713051166, 00:32:46 DEBUG ] 00:32:46 DEBUG 1713051166, 00:32:46 DEBUG "5.937820850674744" 00:32:46 DEBUG } 00:32:46 DEBUG "1808310272" 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 DEBUG ] 00:32:46 DEBUG ] 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 DEBUG } 00:32:46 INFO 00:32:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:32:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:32:46 INFO [loop_until]: OK (rc = 0) 00:32:46 DEBUG --- stdout --- 00:32:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11006m 8986Mi am-7d597cf7d6-8vr8f 10265m 8370Mi am-7d597cf7d6-9wz9h 13854m 9081Mi ds-cts-0 10104m 7028Mi ds-cts-1 10632m 7433Mi ds-cts-2 10630m 7198Mi ds-idrepo-0 11960m 4250Mi ds-idrepo-1 1791m 4286Mi ds-idrepo-2 1744m 4100Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 961Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5935m 1633Mi 00:32:46 DEBUG --- stderr --- 00:32:46 DEBUG 00:32:47 INFO 00:32:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:32:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:32:47 INFO [loop_until]: OK (rc = 0) 00:32:47 DEBUG --- stdout --- 00:32:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4810Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10602m 35% 9267Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11202m 37% 11036Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15625m 52% 2992Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1940m 6% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10259m 34% 10417Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2887Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11101m 37% 9795Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1813m 6% 5430Mi 4% gke-xlou-cdm-ds-0f601751-j488 10720m 35% 8795Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2220Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14899m 49% 10602Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11946m 40% 5635Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6072m 38% 5388Mi 9% 00:32:47 DEBUG --- stderr --- 00:32:47 DEBUG 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "2108.0333333333333" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051225" 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:33:45 ERROR [http_cmd]: ERROR 00:33:45 ERROR --- status code --- 00:33:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:33:45 ERROR --- http response --- 00:33:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:33:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "instance": "10.142.0.107:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "instance": "10.142.0.43:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0.00033333333333333365" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051225" 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG { 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "2102.5333333333333" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "instance": "10.142.0.107:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "instance": "10.142.0.43:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "7323.433333333333" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "ds-idrepo-1" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "ds-idrepo-0" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "instance": "10.142.0.107:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "instance": "10.142.0.43:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0.0005333333333332746" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "ds-idrepo-1" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "ds-cts-1" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0.00006666666666666667" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "ds-idrepo-1" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "ds-cts-1" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "instance": "10.142.0.107:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "instance": "10.142.0.43:9100" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "12578169856" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "2108" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: http status code OK 00:33:45 DEBUG --- status code --- 00:33:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:45 DEBUG --- http response --- 00:33:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:45 DEBUG ----- output ----- 00:33:45 DEBUG { 00:33:45 DEBUG "status": "success", 00:33:45 DEBUG "data": { 00:33:45 DEBUG "resultType": "vector", 00:33:45 DEBUG "result": [ 00:33:45 DEBUG { 00:33:45 DEBUG "metric": { 00:33:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG [...] 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG 00:33:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:45 DEBUG }, 00:33:45 DEBUG "value": [ 00:33:45 DEBUG 1713051225, 00:33:45 DEBUG "0" 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG ] 00:33:45 DEBUG } 00:33:45 DEBUG } 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051225" 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "ds-idrepo-1" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "ds-idrepo-0" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051225, 00:33:46 DEBUG "0" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "ds-idrepo-1" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "ds-idrepo-0" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051225, 00:33:46 DEBUG "0" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051225, 00:33:46 DEBUG "2107.9333333333334" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "11068.60831706951" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG }, 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG "value": [ 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "0" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "0.009" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "379177" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "ds-idrepo-1" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "ds-idrepo-0" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "0" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051226" 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "ds-idrepo-1" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "ds-idrepo-0" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "0" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051226" 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051226" 00:33:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051226" 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "instance": "10.142.0.107:9100" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "instance": "10.142.0.43:9100" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "12610.233333333332" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "1807716352" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "5.8965647404293735" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO [http_cmd]: http status code OK 00:33:46 DEBUG --- status code --- 00:33:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:33:46 DEBUG --- http response --- 00:33:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:33:46 DEBUG ----- output ----- 00:33:46 DEBUG { 00:33:46 DEBUG "status": "success", 00:33:46 DEBUG "data": { 00:33:46 DEBUG "resultType": "vector", 00:33:46 DEBUG "result": [ 00:33:46 DEBUG { 00:33:46 DEBUG "metric": { 00:33:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG [...] 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG 00:33:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:33:46 DEBUG }, 00:33:46 DEBUG "value": [ 00:33:46 DEBUG 1713051226, 00:33:46 DEBUG "5.930802395643071" 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG ] 00:33:46 DEBUG } 00:33:46 DEBUG } 00:33:46 INFO 00:33:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:33:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:33:46 INFO [loop_until]: OK (rc = 0) 00:33:46 DEBUG --- stdout --- 00:33:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11608m 8986Mi am-7d597cf7d6-8vr8f 10346m 8371Mi am-7d597cf7d6-9wz9h 13706m 9072Mi ds-cts-0 10537m 7044Mi ds-cts-1 10577m 7446Mi ds-cts-2 10660m 7202Mi ds-idrepo-0 11937m 4250Mi ds-idrepo-1 1801m 4286Mi ds-idrepo-2 1740m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 8m 961Mi idm-b85f9d978-rl76f 6m 967Mi lodemon-64cbdcc7b4-vt4kk 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5903m 1634Mi 00:33:46 DEBUG --- stderr --- 00:33:46 DEBUG 00:33:47 INFO 00:33:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:33:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:33:47 INFO [loop_until]: OK (rc = 0) 00:33:47 DEBUG --- stdout --- 00:33:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4808Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10107m 33% 9279Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11537m 38% 11042Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15673m 52% 2987Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1951m 6% 6278Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10344m 34% 10422Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11067m 37% 9798Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1836m 6% 5434Mi 4% gke-xlou-cdm-ds-0f601751-j488 10766m 36% 8812Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 68m 0% 2223Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14874m 49% 10589Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 12029m 40% 5646Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6076m 38% 5387Mi 9% 00:33:47 DEBUG --- stderr --- 00:33:47 DEBUG 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051285" 00:34:45 INFO [http_cmd]: http status code OK 00:34:45 DEBUG --- status code --- 00:34:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:45 DEBUG --- http response --- 00:34:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:45 DEBUG ----- output ----- 00:34:45 DEBUG { 00:34:45 DEBUG "status": "success", 00:34:45 DEBUG "data": { 00:34:45 DEBUG "resultType": "vector", 00:34:45 DEBUG "result": [ 00:34:45 DEBUG { 00:34:45 DEBUG "metric": { 00:34:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:45 DEBUG }, 00:34:45 DEBUG "value": [ 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG [...] 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:45 DEBUG }, 00:34:45 DEBUG "value": [ 00:34:45 DEBUG 1713051285, 00:34:45 DEBUG "2105.866666666667" 00:34:45 DEBUG ] 00:34:45 DEBUG } 00:34:45 DEBUG ] 00:34:45 DEBUG } 00:34:45 DEBUG } 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051285" 00:34:45 INFO [http_cmd]: http status code OK 00:34:45 DEBUG --- status code --- 00:34:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:45 DEBUG --- http response --- 00:34:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:34:45 ERROR [http_cmd]: ERROR 00:34:45 ERROR --- status code --- 00:34:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:34:45 ERROR --- http response --- 00:34:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:34:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:34:45 INFO [http_cmd]: http status code OK 00:34:45 DEBUG --- status code --- 00:34:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:45 DEBUG --- http response --- 00:34:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:45 DEBUG ----- output ----- 00:34:45 DEBUG { 00:34:45 DEBUG "status": "success", 00:34:45 DEBUG "data": { 00:34:45 DEBUG "resultType": "vector", 00:34:45 DEBUG "result": [ 00:34:45 DEBUG { 00:34:45 DEBUG "metric": { 00:34:45 DEBUG "instance": "10.142.0.107:9100" 00:34:45 DEBUG }, 00:34:45 DEBUG "value": [ 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG [...] 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG 00:34:45 DEBUG "instance": "10.142.0.43:9100" 00:34:45 DEBUG }, 00:34:45 DEBUG "value": [ 00:34:45 DEBUG 1713051285, 00:34:45 DEBUG "0" 00:34:45 DEBUG ] 00:34:45 DEBUG } 00:34:45 DEBUG ] 00:34:45 DEBUG } 00:34:45 DEBUG } 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051285" 00:34:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051285" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG --- status code --- 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG { 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG "data": { 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG "result": [ 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG { 00:34:46 DEBUG { 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG }, 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "result": [ 00:34:46 DEBUG "value": [ 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG { 00:34:46 DEBUG 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG "metric": { 00:34:46 DEBUG 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "instance": "10.142.0.107:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 DEBUG 00:34:46 DEBUG "instance": "10.142.0.107:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG }, 00:34:46 DEBUG [...] 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG { 00:34:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG }, 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "data": { 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG 1713051285, 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "ds-cts-1" 00:34:46 DEBUG 00:34:46 DEBUG "result": [ 00:34:46 DEBUG "2102.133333333333" 00:34:46 DEBUG 00:34:46 DEBUG "instance": "10.142.0.43:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG "pod": "ds-idrepo-0" 00:34:46 DEBUG { 00:34:46 DEBUG ] 00:34:46 DEBUG "instance": "10.142.0.43:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG }, 00:34:46 DEBUG "metric": { 00:34:46 DEBUG } 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051285, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG "instance": "10.142.0.107:9100" 00:34:46 DEBUG ] 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051285, 00:34:46 DEBUG "0.00009659722222222223" 00:34:46 DEBUG 1713051285, 00:34:46 DEBUG }, 00:34:46 DEBUG } 00:34:46 DEBUG 1713051285, 00:34:46 DEBUG "0.0004333333333332708" 00:34:46 DEBUG ] 00:34:46 DEBUG "0" 00:34:46 DEBUG "value": [ 00:34:46 DEBUG } 00:34:46 DEBUG "5589.166666666666" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 DEBUG [...] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 DEBUG 00:34:46 DEBUG } 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "instance": "10.142.0.43:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051285, 00:34:46 DEBUG "12578728960" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "ds-cts-1" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "ds-idrepo-0" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "5579529.138167032" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "2105.6" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0.0095" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "ds-idrepo-0" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "2105.1" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "305289904.9082515" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "instance": "10.142.0.107:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "instance": "10.142.0.43:9100" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "12244.766666666665" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "379271" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "ds-idrepo-0" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "ds-idrepo-1" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "ds-idrepo-0" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "0" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051286" 00:34:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051286" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "1811640320" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:34:46 INFO [http_cmd]: http status code OK 00:34:46 DEBUG }, 00:34:46 DEBUG --- status code --- 00:34:46 DEBUG "value": [ 00:34:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:34:46 DEBUG 00:34:46 DEBUG --- http response --- 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "5.790107366553022" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:34:46 DEBUG ----- output ----- 00:34:46 DEBUG { 00:34:46 DEBUG "status": "success", 00:34:46 DEBUG "data": { 00:34:46 DEBUG "resultType": "vector", 00:34:46 DEBUG "result": [ 00:34:46 DEBUG { 00:34:46 DEBUG "metric": { 00:34:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG [...] 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG 00:34:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:34:46 DEBUG }, 00:34:46 DEBUG "value": [ 00:34:46 DEBUG 1713051286, 00:34:46 DEBUG "5.944300932819722" 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG ] 00:34:46 DEBUG } 00:34:46 DEBUG } 00:34:46 INFO 00:34:46 INFO [loop_until]: kubectl --namespace=xlou top pods 00:34:46 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:34:46 INFO [loop_until]: OK (rc = 0) 00:34:46 DEBUG --- stdout --- 00:34:46 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11289m 8986Mi am-7d597cf7d6-8vr8f 9960m 8370Mi am-7d597cf7d6-9wz9h 13606m 9076Mi ds-cts-0 10043m 7158Mi ds-cts-1 10418m 7423Mi ds-cts-2 10851m 7206Mi ds-idrepo-0 11734m 4250Mi ds-idrepo-1 1763m 4286Mi ds-idrepo-2 1727m 4103Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 7m 961Mi idm-b85f9d978-rl76f 4m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6087m 1635Mi 00:34:46 DEBUG --- stderr --- 00:34:46 DEBUG 00:34:47 INFO 00:34:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:34:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:34:47 INFO [loop_until]: OK (rc = 0) 00:34:47 DEBUG --- stdout --- 00:34:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4805Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10538m 35% 9276Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11550m 38% 11041Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15619m 52% 2988Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1974m 6% 6276Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10193m 34% 10558Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2882Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11080m 37% 9797Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1814m 6% 5440Mi 4% gke-xlou-cdm-ds-0f601751-j488 10645m 35% 8820Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2213Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14927m 49% 10600Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11975m 40% 5641Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6060m 38% 5388Mi 9% 00:34:47 DEBUG --- stderr --- 00:34:47 DEBUG 00:35:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051345" 00:35:45 INFO [http_cmd]: http status code OK 00:35:45 DEBUG --- status code --- 00:35:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:45 DEBUG --- http response --- 00:35:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:45 DEBUG ----- output ----- 00:35:45 DEBUG { 00:35:45 DEBUG "status": "success", 00:35:45 DEBUG "data": { 00:35:45 DEBUG "resultType": "vector", 00:35:45 DEBUG "result": [ 00:35:45 DEBUG { 00:35:45 DEBUG "metric": { 00:35:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:45 DEBUG }, 00:35:45 DEBUG "value": [ 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG [...] 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:45 DEBUG }, 00:35:45 DEBUG "value": [ 00:35:45 DEBUG 1713051345, 00:35:45 DEBUG "2109.866666666667" 00:35:45 DEBUG ] 00:35:45 DEBUG } 00:35:45 DEBUG ] 00:35:45 DEBUG } 00:35:45 DEBUG } 00:35:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051345" 00:35:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051345" 00:35:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051345" 00:35:45 INFO [http_cmd]: http status code OK 00:35:45 DEBUG --- status code --- 00:35:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:45 DEBUG --- http response --- 00:35:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:35:45 ERROR [http_cmd]: ERROR 00:35:45 ERROR --- status code --- 00:35:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:35:45 ERROR --- http response --- 00:35:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:35:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:35:45 INFO [http_cmd]: http status code OK 00:35:45 DEBUG --- status code --- 00:35:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:45 DEBUG --- http response --- 00:35:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:45 DEBUG ----- output ----- 00:35:45 DEBUG { 00:35:45 DEBUG "status": "success", 00:35:45 DEBUG "data": { 00:35:45 DEBUG "resultType": "vector", 00:35:45 DEBUG "result": [ 00:35:45 DEBUG { 00:35:45 DEBUG "metric": { 00:35:45 DEBUG "instance": "10.142.0.107:9100" 00:35:45 DEBUG }, 00:35:45 DEBUG "value": [ 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG [...] 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG 00:35:45 DEBUG "instance": "10.142.0.43:9100" 00:35:45 DEBUG }, 00:35:45 DEBUG "value": [ 00:35:45 DEBUG 1713051345, 00:35:45 DEBUG "0.0003333333333333262" 00:35:45 DEBUG ] 00:35:45 DEBUG } 00:35:45 DEBUG ] 00:35:45 DEBUG } 00:35:45 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "instance": "10.142.0.107:9100" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "instance": "10.142.0.43:9100" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "6880.233333333334" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "ds-idrepo-0" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "0" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "2112" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "instance": "10.142.0.107:9100" 00:35:46 DEBUG "instance": "10.142.0.107:9100" 00:35:46 DEBUG }, 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "instance": "10.142.0.43:9100" 00:35:46 DEBUG "instance": "10.142.0.43:9100" 00:35:46 DEBUG }, 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "0.00043333333333350765" 00:35:46 DEBUG "12579312640" 00:35:46 DEBUG ] 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG { 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG { 00:35:46 DEBUG "data": { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG { 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "metric": { 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG { 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG "metric": { 00:35:46 DEBUG }, 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG }, 00:35:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:35:46 DEBUG "value": [ 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG }, 00:35:46 DEBUG 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "ds-idrepo-0" 00:35:46 DEBUG 00:35:46 DEBUG "pod": "ds-cts-1" 00:35:46 DEBUG 00:35:46 DEBUG }, 00:35:46 DEBUG "pod": "ds-cts-1" 00:35:46 DEBUG }, 00:35:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:35:46 DEBUG "value": [ 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG }, 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG "0" 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "0.00009295833333333333" 00:35:46 DEBUG 1713051346, 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG ] 00:35:46 DEBUG "0" 00:35:46 DEBUG ] 00:35:46 DEBUG "0" 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "2109.633333333333" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG { 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG { 00:35:46 DEBUG { 00:35:46 DEBUG "data": { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "data": { 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG { 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "result": [ 00:35:46 DEBUG "metric": { 00:35:46 DEBUG { 00:35:46 DEBUG { 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG }, 00:35:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:46 DEBUG "value": [ 00:35:46 DEBUG }, 00:35:46 DEBUG }, 00:35:46 DEBUG 00:35:46 DEBUG "value": [ 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "ds-idrepo-0" 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG }, 00:35:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:46 DEBUG "value": [ 00:35:46 DEBUG }, 00:35:46 DEBUG }, 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG "value": [ 00:35:46 DEBUG "0" 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG ] 00:35:46 DEBUG "0" 00:35:46 DEBUG "0.01" 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG ] 00:35:46 DEBUG ] 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "ds-idrepo-0" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "0" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "379486.5" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "2110.1666666666665" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "27691.349285006374" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "0" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "instance": "10.142.0.107:9100" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "instance": "10.142.0.43:9100" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "12183.599999999999" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "5.906627862114052" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "ds-idrepo-1" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "ds-idrepo-0" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "0" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051346" 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "1814691840" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:46 INFO [http_cmd]: http status code OK 00:35:46 DEBUG --- status code --- 00:35:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:35:46 DEBUG --- http response --- 00:35:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:35:46 DEBUG ----- output ----- 00:35:46 DEBUG { 00:35:46 DEBUG "status": "success", 00:35:46 DEBUG "data": { 00:35:46 DEBUG "resultType": "vector", 00:35:46 DEBUG "result": [ 00:35:46 DEBUG { 00:35:46 DEBUG "metric": { 00:35:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG [...] 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG 00:35:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:35:46 DEBUG }, 00:35:46 DEBUG "value": [ 00:35:46 DEBUG 1713051346, 00:35:46 DEBUG "5.875410766671371" 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG ] 00:35:46 DEBUG } 00:35:46 DEBUG } 00:35:47 INFO 00:35:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:35:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:35:47 INFO [loop_until]: OK (rc = 0) 00:35:47 DEBUG --- stdout --- 00:35:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11295m 8988Mi am-7d597cf7d6-8vr8f 10143m 8375Mi am-7d597cf7d6-9wz9h 13720m 9077Mi ds-cts-0 10253m 7028Mi ds-cts-1 10553m 7434Mi ds-cts-2 10891m 7234Mi ds-idrepo-0 11988m 4250Mi ds-idrepo-1 1884m 4286Mi ds-idrepo-2 1704m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 8m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 7m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5724m 1637Mi 00:35:47 DEBUG --- stderr --- 00:35:47 DEBUG 00:35:47 INFO 00:35:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:35:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:35:47 INFO [loop_until]: OK (rc = 0) 00:35:47 DEBUG --- stdout --- 00:35:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4810Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10570m 35% 9289Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11716m 39% 11050Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15625m 52% 2976Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1913m 6% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10095m 33% 10429Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2886Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10659m 35% 9804Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1846m 6% 5429Mi 4% gke-xlou-cdm-ds-0f601751-j488 10627m 35% 8815Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2219Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14900m 49% 10605Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11938m 39% 5638Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6026m 37% 5389Mi 9% 00:35:47 DEBUG --- stderr --- 00:35:47 DEBUG 00:36:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051405" 00:36:45 INFO [http_cmd]: http status code OK 00:36:45 DEBUG --- status code --- 00:36:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:45 DEBUG --- http response --- 00:36:45 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:36:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051405" 00:36:45 INFO [http_cmd]: http status code OK 00:36:45 DEBUG --- status code --- 00:36:45 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:45 DEBUG --- http response --- 00:36:45 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:45 DEBUG ----- output ----- 00:36:45 DEBUG { 00:36:45 DEBUG "status": "success", 00:36:45 DEBUG "data": { 00:36:45 DEBUG "resultType": "vector", 00:36:45 DEBUG "result": [ 00:36:45 DEBUG { 00:36:45 DEBUG "metric": { 00:36:45 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:45 DEBUG }, 00:36:45 DEBUG "value": [ 00:36:45 DEBUG 00:36:45 DEBUG 00:36:45 DEBUG 00:36:45 DEBUG [...] 00:36:45 DEBUG 00:36:45 DEBUG 00:36:45 DEBUG 00:36:45 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:45 DEBUG }, 00:36:45 DEBUG "value": [ 00:36:45 DEBUG 1713051405, 00:36:45 DEBUG "2089.3333333333335" 00:36:45 DEBUG ] 00:36:45 DEBUG } 00:36:45 DEBUG ] 00:36:45 DEBUG } 00:36:45 DEBUG } 00:36:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051405" 00:36:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051405" 00:36:45 ERROR [http_cmd]: ERROR 00:36:45 ERROR --- status code --- 00:36:45 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:36:45 ERROR --- http response --- 00:36:45 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:36:45 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "instance": "10.142.0.107:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "instance": "10.142.0.43:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051405, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "instance": "10.142.0.107:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "instance": "10.142.0.43:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "5290.066666666667" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "instance": "10.142.0.107:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "instance": "10.142.0.43:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "12579949568" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG { 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG "metric": { 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG "instance": "10.142.0.107:9100" 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG }, 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:36:46 DEBUG "value": [ 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG [...] 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "instance": "10.142.0.43:9100" 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG "0.0005666666666665302" 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG ] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG } 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG ] 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG } 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG } 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:46 DEBUG }, 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG "2106.9666666666667" 00:36:46 DEBUG ] 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-cts-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG { 00:36:46 DEBUG { 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG "data": { 00:36:46 DEBUG "data": { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG "result": [ 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG { 00:36:46 DEBUG { 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "metric": { 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG "data": { 00:36:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:46 DEBUG }, 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG }, 00:36:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG "result": [ 00:36:46 DEBUG "value": [ 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG { 00:36:46 DEBUG 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "metric": { 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG }, 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-cts-1" 00:36:46 DEBUG 00:36:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:46 DEBUG 00:36:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:46 DEBUG }, 00:36:46 DEBUG [...] 00:36:46 DEBUG }, 00:36:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG "value": [ 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0.00007308333333333334" 00:36:46 DEBUG 00:36:46 DEBUG "0" 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "2089.2666666666664" 00:36:46 DEBUG ] 00:36:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:46 DEBUG ] 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG }, 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG "value": [ 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG "0.01" 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "379136" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "2089.5666666666666" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "instance": "10.142.0.107:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "instance": "10.142.0.43:9100" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "11802.766666666665" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "5.894825748129497" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "ds-idrepo-1" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051406" 00:36:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051406" 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "1813397504" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:46 INFO [http_cmd]: http status code OK 00:36:46 DEBUG --- status code --- 00:36:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:36:46 DEBUG --- http response --- 00:36:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:36:46 DEBUG ----- output ----- 00:36:46 DEBUG { 00:36:46 DEBUG "status": "success", 00:36:46 DEBUG "data": { 00:36:46 DEBUG "resultType": "vector", 00:36:46 DEBUG "result": [ 00:36:46 DEBUG { 00:36:46 DEBUG "metric": { 00:36:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG [...] 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG 00:36:46 DEBUG "pod": "ds-idrepo-0" 00:36:46 DEBUG }, 00:36:46 DEBUG "value": [ 00:36:46 DEBUG 1713051406, 00:36:46 DEBUG "0" 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG ] 00:36:46 DEBUG } 00:36:46 DEBUG } 00:36:47 INFO 00:36:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:36:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:36:47 INFO [loop_until]: OK (rc = 0) 00:36:47 DEBUG --- stdout --- 00:36:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10922m 8987Mi am-7d597cf7d6-8vr8f 10324m 8376Mi am-7d597cf7d6-9wz9h 13500m 9073Mi ds-cts-0 9929m 7049Mi ds-cts-1 10480m 7430Mi ds-cts-2 10871m 7233Mi ds-idrepo-0 12024m 4251Mi ds-idrepo-1 1842m 4286Mi ds-idrepo-2 1769m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 962Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6001m 1638Mi 00:36:47 DEBUG --- stderr --- 00:36:47 DEBUG 00:36:47 INFO 00:36:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:36:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:36:47 INFO [loop_until]: OK (rc = 0) 00:36:47 DEBUG --- stdout --- 00:36:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4804Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10661m 35% 9286Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11436m 38% 11045Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15567m 52% 2975Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1905m 6% 6273Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10114m 33% 10421Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2883Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11108m 37% 9806Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1839m 6% 5436Mi 4% gke-xlou-cdm-ds-0f601751-j488 10599m 35% 8822Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2206Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14709m 49% 10602Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11783m 39% 5627Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6034m 37% 5388Mi 9% 00:36:47 DEBUG --- stderr --- 00:36:47 DEBUG 00:37:45 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051465" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "2091.9333333333334" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "instance": "10.142.0.107:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "instance": "10.142.0.43:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051466" 00:37:46 ERROR [http_cmd]: ERROR 00:37:46 ERROR --- status code --- 00:37:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:37:46 ERROR --- http response --- 00:37:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:37:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "instance": "10.142.0.107:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "instance": "10.142.0.43:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "12580555776" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "instance": "10.142.0.107:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "instance": "10.142.0.43:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "5293.733333333333" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG { 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "instance": "10.142.0.107:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "instance": "10.142.0.43:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0.0004333333333332708" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "2105.2666666666664" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG --- status code --- 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 DEBUG ----- output ----- 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG { 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG "data": { 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG "result": [ 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:37:46 DEBUG { 00:37:46 DEBUG { 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "metric": { 00:37:46 DEBUG { 00:37:46 DEBUG "data": { 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG }, 00:37:46 DEBUG "data": { 00:37:46 DEBUG "result": [ 00:37:46 DEBUG "value": [ 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG { 00:37:46 DEBUG 00:37:46 DEBUG "result": [ 00:37:46 DEBUG "metric": { 00:37:46 DEBUG 00:37:46 DEBUG { 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG 00:37:46 DEBUG "metric": { 00:37:46 DEBUG }, 00:37:46 DEBUG [...] 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG }, 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG }, 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "value": [ 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "0" 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-cts-1" 00:37:46 DEBUG ] 00:37:46 DEBUG 00:37:46 DEBUG }, 00:37:46 DEBUG } 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG "value": [ 00:37:46 DEBUG ] 00:37:46 DEBUG }, 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG } 00:37:46 DEBUG "value": [ 00:37:46 DEBUG "0" 00:37:46 DEBUG } 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG ] 00:37:46 DEBUG "0" 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG } 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG { 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0.01" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-cts-1" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0.0001666666666666667" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "2092.0333333333333" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051466" 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG 00:37:46 DEBUG { 00:37:46 DEBUG [...] 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG "data": { 00:37:46 DEBUG 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG "result": [ 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG { 00:37:46 DEBUG }, 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "value": [ 00:37:46 DEBUG { 00:37:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG }, 00:37:46 DEBUG "378277" 00:37:46 DEBUG "data": { 00:37:46 DEBUG "value": [ 00:37:46 DEBUG ] 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG 00:37:46 DEBUG } 00:37:46 DEBUG "result": [ 00:37:46 DEBUG 00:37:46 DEBUG ] 00:37:46 DEBUG { 00:37:46 DEBUG 00:37:46 DEBUG } 00:37:46 DEBUG "metric": { 00:37:46 DEBUG [...] 00:37:46 DEBUG } 00:37:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:37:46 DEBUG 00:37:46 DEBUG }, 00:37:46 DEBUG 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:37:46 DEBUG 00:37:46 DEBUG }, 00:37:46 DEBUG 00:37:46 DEBUG "value": [ 00:37:46 DEBUG [...] 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG 00:37:46 DEBUG "2092.5333333333333" 00:37:46 DEBUG 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG ] 00:37:46 DEBUG 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG } 00:37:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG ] 00:37:46 DEBUG }, 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG } 00:37:46 DEBUG "value": [ 00:37:46 DEBUG } 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG "5.907160213498983" 00:37:46 DEBUG { 00:37:46 DEBUG ] 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG } 00:37:46 DEBUG "data": { 00:37:46 DEBUG ] 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG } 00:37:46 DEBUG "result": [ 00:37:46 DEBUG } 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "instance": "10.142.0.107:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "instance": "10.142.0.43:9100" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "11902.099999999999" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "pod": "ds-idrepo-1" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "pod": "ds-idrepo-0" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "0" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051466" 00:37:46 INFO [http_cmd]: http status code OK 00:37:46 DEBUG --- status code --- 00:37:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:37:46 DEBUG --- http response --- 00:37:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:37:46 DEBUG ----- output ----- 00:37:46 DEBUG { 00:37:46 DEBUG "status": "success", 00:37:46 DEBUG "data": { 00:37:46 DEBUG "resultType": "vector", 00:37:46 DEBUG "result": [ 00:37:46 DEBUG { 00:37:46 DEBUG "metric": { 00:37:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG [...] 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG 00:37:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:37:46 DEBUG }, 00:37:46 DEBUG "value": [ 00:37:46 DEBUG 1713051466, 00:37:46 DEBUG "1814958080" 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG ] 00:37:46 DEBUG } 00:37:46 DEBUG } 00:37:47 INFO 00:37:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:37:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:37:47 INFO [loop_until]: OK (rc = 0) 00:37:47 DEBUG --- stdout --- 00:37:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11018m 8984Mi am-7d597cf7d6-8vr8f 10184m 8377Mi am-7d597cf7d6-9wz9h 13666m 9083Mi ds-cts-0 10323m 7040Mi ds-cts-1 10719m 7460Mi ds-cts-2 10488m 7304Mi ds-idrepo-0 11638m 4251Mi ds-idrepo-1 1802m 4286Mi ds-idrepo-2 1767m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 962Mi idm-b85f9d978-rl76f 10m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5755m 1639Mi 00:37:47 DEBUG --- stderr --- 00:37:47 DEBUG 00:37:47 INFO 00:37:47 INFO [loop_until]: kubectl --namespace=xlou top node 00:37:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:37:47 INFO [loop_until]: OK (rc = 0) 00:37:47 DEBUG --- stdout --- 00:37:47 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 72m 0% 4811Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10620m 35% 9366Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11503m 38% 11043Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15546m 52% 2974Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1944m 6% 6278Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10340m 34% 10439Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2877Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11036m 36% 9797Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1785m 5% 5442Mi 4% gke-xlou-cdm-ds-0f601751-j488 10896m 36% 8838Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2216Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14658m 49% 10597Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11926m 39% 5636Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6031m 37% 5388Mi 9% 00:37:47 DEBUG --- stderr --- 00:37:47 DEBUG 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051526" 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "2102.2" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "instance": "10.142.0.107:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "instance": "10.142.0.43:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0.00033333333333333365" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051526" 00:38:46 ERROR [http_cmd]: ERROR 00:38:46 ERROR --- status code --- 00:38:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:38:46 ERROR --- http response --- 00:38:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:38:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "instance": "10.142.0.107:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "instance": "10.142.0.43:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "12581252096" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "instance": "10.142.0.107:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "instance": "10.142.0.43:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "6851.7" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-idrepo-0" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-idrepo-0" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-idrepo-0" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "2110.1666666666665" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "instance": "10.142.0.107:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "instance": "10.142.0.43:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0.0003999999999997783" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0.010499999999999999" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG { 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-cts-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-idrepo-0" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "2104.866666666667" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "5.892602689601967" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-cts-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0.00023529166666666664" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "19879.498004078" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "377700" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "2104.9666666666667" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "instance": "10.142.0.107:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "instance": "10.142.0.43:9100" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "12124.199999999997" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "5.859078190316202" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "pod": "ds-idrepo-1" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "pod": "ds-idrepo-0" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "0" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051526" 00:38:46 INFO [http_cmd]: http status code OK 00:38:46 DEBUG --- status code --- 00:38:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:38:46 DEBUG --- http response --- 00:38:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:38:46 DEBUG ----- output ----- 00:38:46 DEBUG { 00:38:46 DEBUG "status": "success", 00:38:46 DEBUG "data": { 00:38:46 DEBUG "resultType": "vector", 00:38:46 DEBUG "result": [ 00:38:46 DEBUG { 00:38:46 DEBUG "metric": { 00:38:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG [...] 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG 00:38:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:38:46 DEBUG }, 00:38:46 DEBUG "value": [ 00:38:46 DEBUG 1713051526, 00:38:46 DEBUG "1817239552" 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG ] 00:38:46 DEBUG } 00:38:46 DEBUG } 00:38:47 INFO 00:38:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:38:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:38:47 INFO [loop_until]: OK (rc = 0) 00:38:47 DEBUG --- stdout --- 00:38:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11193m 9031Mi am-7d597cf7d6-8vr8f 10351m 8385Mi am-7d597cf7d6-9wz9h 13545m 9079Mi ds-cts-0 10316m 7131Mi ds-cts-1 10307m 7449Mi ds-cts-2 10531m 7301Mi ds-idrepo-0 11752m 4251Mi ds-idrepo-1 1789m 4286Mi ds-idrepo-2 1674m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 962Mi idm-b85f9d978-rl76f 7m 967Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5954m 1641Mi 00:38:47 DEBUG --- stderr --- 00:38:47 DEBUG 00:38:48 INFO 00:38:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:38:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:38:48 INFO [loop_until]: OK (rc = 0) 00:38:48 DEBUG --- stdout --- 00:38:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 4802Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10512m 35% 9382Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11657m 39% 11085Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15563m 52% 2975Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1929m 6% 6272Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10021m 33% 10524Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2878Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11030m 36% 9819Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1769m 5% 5444Mi 4% gke-xlou-cdm-ds-0f601751-j488 10481m 35% 8834Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2216Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14685m 49% 10614Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11891m 39% 5641Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 5833m 36% 5389Mi 9% 00:38:48 DEBUG --- stderr --- 00:38:48 DEBUG 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 ERROR [http_cmd]: ERROR 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 ERROR --- status code --- 00:39:46 DEBUG --- status code --- 00:39:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 ERROR --- http response --- 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:39:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:39:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "2125.5666666666666" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "instance": "10.142.0.107:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "instance": "10.142.0.43:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "12581813248" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "instance": "10.142.0.107:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "instance": "10.142.0.43:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "instance": "10.142.0.107:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "instance": "10.142.0.43:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "5306.233333333333" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "2090.3" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-2" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "5280.316268943192" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0.011" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "2125.7" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-cts-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051586" 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "instance": "10.142.0.107:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "instance": "10.142.0.43:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0.00040000000000001514" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "5.964966498252917" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-cts-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0.0001" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-2" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "378242" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "2125.5" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "instance": "10.142.0.107:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "instance": "10.142.0.43:9100" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "12229.399999999998" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "ds-idrepo-1" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051586" 00:39:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051586" 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "1819250688" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:46 INFO [http_cmd]: http status code OK 00:39:46 DEBUG --- status code --- 00:39:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:39:46 DEBUG --- http response --- 00:39:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:39:46 DEBUG ----- output ----- 00:39:46 DEBUG { 00:39:46 DEBUG "status": "success", 00:39:46 DEBUG "data": { 00:39:46 DEBUG "resultType": "vector", 00:39:46 DEBUG "result": [ 00:39:46 DEBUG { 00:39:46 DEBUG "metric": { 00:39:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG [...] 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG 00:39:46 DEBUG "pod": "ds-idrepo-0" 00:39:46 DEBUG }, 00:39:46 DEBUG "value": [ 00:39:46 DEBUG 1713051586, 00:39:46 DEBUG "0" 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG ] 00:39:46 DEBUG } 00:39:46 DEBUG } 00:39:47 INFO 00:39:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:39:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:39:47 INFO [loop_until]: OK (rc = 0) 00:39:47 DEBUG --- stdout --- 00:39:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11293m 8983Mi am-7d597cf7d6-8vr8f 9819m 8382Mi am-7d597cf7d6-9wz9h 13862m 9082Mi ds-cts-0 10205m 7126Mi ds-cts-1 10772m 7440Mi ds-cts-2 10740m 7315Mi ds-idrepo-0 12072m 4250Mi ds-idrepo-1 1853m 4286Mi ds-idrepo-2 1675m 4102Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 963Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 7m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6073m 1642Mi 00:39:47 DEBUG --- stderr --- 00:39:47 DEBUG 00:39:48 INFO 00:39:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:39:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:39:48 INFO [loop_until]: OK (rc = 0) 00:39:48 DEBUG --- stdout --- 00:39:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4799Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10528m 35% 9391Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11681m 39% 11052Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15674m 52% 2979Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1978m 6% 6278Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10206m 34% 10530Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2885Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11109m 37% 9809Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1777m 5% 5420Mi 4% gke-xlou-cdm-ds-0f601751-j488 10656m 35% 8845Mi 7% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2223Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14904m 49% 10606Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11492m 38% 5643Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6076m 38% 5390Mi 9% 00:39:48 DEBUG --- stderr --- 00:39:48 DEBUG 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051646" 00:40:46 ERROR [http_cmd]: ERROR 00:40:46 ERROR --- status code --- 00:40:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:40:46 ERROR --- http response --- 00:40:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:40:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "2113.4333333333334" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "instance": "10.142.0.107:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "instance": "10.142.0.43:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "12582450176" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "instance": "10.142.0.107:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "instance": "10.142.0.43:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "instance": "10.142.0.107:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "instance": "10.142.0.43:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "5405" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-cts-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "instance": "10.142.0.107:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "instance": "10.142.0.43:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0.0004333333333332708" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "2092.4666666666667" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "2113.5" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0.011" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "5.88831968929744" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-cts-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0.00014937499999999999" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "379696" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "2113.6" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "instance": "10.142.0.107:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "instance": "10.142.0.43:9100" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "12026.366666666665" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "ds-idrepo-1" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051646" 00:40:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051646" 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "1822101504" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:46 INFO [http_cmd]: http status code OK 00:40:46 DEBUG --- status code --- 00:40:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:40:46 DEBUG --- http response --- 00:40:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:40:46 DEBUG ----- output ----- 00:40:46 DEBUG { 00:40:46 DEBUG "status": "success", 00:40:46 DEBUG "data": { 00:40:46 DEBUG "resultType": "vector", 00:40:46 DEBUG "result": [ 00:40:46 DEBUG { 00:40:46 DEBUG "metric": { 00:40:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG [...] 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG 00:40:46 DEBUG "pod": "ds-idrepo-0" 00:40:46 DEBUG }, 00:40:46 DEBUG "value": [ 00:40:46 DEBUG 1713051646, 00:40:46 DEBUG "0" 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG ] 00:40:46 DEBUG } 00:40:46 DEBUG } 00:40:47 INFO 00:40:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:40:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:40:47 INFO [loop_until]: OK (rc = 0) 00:40:47 DEBUG --- stdout --- 00:40:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11241m 9020Mi am-7d597cf7d6-8vr8f 10128m 8376Mi am-7d597cf7d6-9wz9h 13937m 9078Mi ds-cts-0 9771m 7137Mi ds-cts-1 10676m 7575Mi ds-cts-2 10082m 7313Mi ds-idrepo-0 11937m 4268Mi ds-idrepo-1 1821m 4286Mi ds-idrepo-2 1763m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 963Mi idm-b85f9d978-rl76f 5m 967Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5686m 1643Mi 00:40:47 DEBUG --- stderr --- 00:40:47 DEBUG 00:40:48 INFO 00:40:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:40:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:40:48 INFO [loop_until]: OK (rc = 0) 00:40:48 DEBUG --- stdout --- 00:40:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 70m 0% 4801Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10428m 34% 9397Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11452m 38% 11075Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15019m 50% 2985Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1927m 6% 6284Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10030m 33% 10536Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2888Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10962m 36% 9791Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1799m 6% 5434Mi 4% gke-xlou-cdm-ds-0f601751-j488 10951m 36% 8966Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2214Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14729m 49% 10609Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11911m 39% 5658Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6026m 37% 5392Mi 9% 00:40:48 DEBUG --- stderr --- 00:40:48 DEBUG 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 ERROR [http_cmd]: ERROR 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 ERROR --- status code --- 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 ERROR --- http response --- 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:41:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:41:46 DEBUG ----- output ----- 00:41:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "instance": "10.142.0.107:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "instance": "10.142.0.43:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "12583023616" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "instance": "10.142.0.107:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "instance": "10.142.0.43:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0.00033333333333333365" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "2091.4" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "instance": "10.142.0.107:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "instance": "10.142.0.43:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0.0004333333333332708" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "instance": "10.142.0.107:9100" 00:41:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:41:46 DEBUG }, 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "instance": "10.142.0.43:9100" 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "5643.633333333333" 00:41:46 DEBUG "4445413.458439279" 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG { 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:41:46 DEBUG "data": { 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "data": { 00:41:46 DEBUG "data": { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "data": { 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG "result": [ 00:41:46 DEBUG "result": [ 00:41:46 DEBUG }, 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG "result": [ 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG "value": [ 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG { 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG }, 00:41:46 DEBUG }, 00:41:46 DEBUG }, 00:41:46 DEBUG 00:41:46 DEBUG }, 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG "value": [ 00:41:46 DEBUG "value": [ 00:41:46 DEBUG [...] 00:41:46 DEBUG "value": [ 00:41:46 DEBUG { 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "data": { 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG [...] 00:41:46 DEBUG [...] 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG [...] 00:41:46 DEBUG "result": [ 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG }, 00:41:46 DEBUG 00:41:46 DEBUG { 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG "metric": { 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG 00:41:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG "pod": "ds-idrepo-0" 00:41:46 DEBUG "pod": "ds-idrepo-0" 00:41:46 DEBUG "0" 00:41:46 DEBUG "pod": "ds-cts-1" 00:41:46 DEBUG }, 00:41:46 DEBUG "pod": "ds-idrepo-0" 00:41:46 DEBUG }, 00:41:46 DEBUG }, 00:41:46 DEBUG }, 00:41:46 DEBUG ] 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG "value": [ 00:41:46 DEBUG "value": [ 00:41:46 DEBUG } 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG ] 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "2104.6666666666665" 00:41:46 DEBUG "0" 00:41:46 DEBUG "0" 00:41:46 DEBUG } 00:41:46 DEBUG "0" 00:41:46 DEBUG 00:41:46 DEBUG "0" 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG [...] 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG ] 00:41:46 DEBUG 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG } 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "337654419.36754644" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "2091.5" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0.011" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "ds-idrepo-0" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "5.9165023787151405" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "ds-cts-1" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "379985.5" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "2092.233333333333" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "instance": "10.142.0.107:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "instance": "10.142.0.43:9100" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "12275.83333333333" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "ds-idrepo-1" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "ds-idrepo-0" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "0" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051706" 00:41:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051706" 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "11.24518425670594" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:46 INFO [http_cmd]: http status code OK 00:41:46 DEBUG --- status code --- 00:41:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:41:46 DEBUG --- http response --- 00:41:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:41:46 DEBUG ----- output ----- 00:41:46 DEBUG { 00:41:46 DEBUG "status": "success", 00:41:46 DEBUG "data": { 00:41:46 DEBUG "resultType": "vector", 00:41:46 DEBUG "result": [ 00:41:46 DEBUG { 00:41:46 DEBUG "metric": { 00:41:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG [...] 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG 00:41:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:41:46 DEBUG }, 00:41:46 DEBUG "value": [ 00:41:46 DEBUG 1713051706, 00:41:46 DEBUG "1822138368" 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG ] 00:41:46 DEBUG } 00:41:46 DEBUG } 00:41:47 INFO 00:41:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:41:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:41:47 INFO [loop_until]: OK (rc = 0) 00:41:47 DEBUG --- stdout --- 00:41:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11319m 8989Mi am-7d597cf7d6-8vr8f 9918m 8381Mi am-7d597cf7d6-9wz9h 13876m 9081Mi ds-cts-0 10013m 7137Mi ds-cts-1 10690m 7558Mi ds-cts-2 9975m 7333Mi ds-idrepo-0 11998m 4268Mi ds-idrepo-1 1785m 4291Mi ds-idrepo-2 1687m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 9m 963Mi idm-b85f9d978-rl76f 7m 968Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5687m 1645Mi 00:41:47 DEBUG --- stderr --- 00:41:47 DEBUG 00:41:48 INFO 00:41:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:41:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:41:48 INFO [loop_until]: OK (rc = 0) 00:41:48 DEBUG --- stdout --- 00:41:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 4797Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10151m 34% 9405Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11559m 38% 11050Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15518m 51% 2979Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1888m 6% 6291Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 9958m 33% 10541Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2880Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11009m 36% 9807Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1773m 5% 5447Mi 4% gke-xlou-cdm-ds-0f601751-j488 10611m 35% 8970Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2211Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14710m 49% 10616Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11829m 39% 5663Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6014m 37% 5390Mi 9% 00:41:48 DEBUG --- stderr --- 00:41:48 DEBUG 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051766" 00:42:46 ERROR [http_cmd]: ERROR 00:42:46 ERROR --- status code --- 00:42:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:42:46 ERROR --- http response --- 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:42:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "instance": "10.142.0.107:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "instance": "10.142.0.43:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "12583654400" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "2107.4666666666667" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "instance": "10.142.0.107:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "instance": "10.142.0.43:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "instance": "10.142.0.107:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "instance": "10.142.0.43:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "5309.566666666667" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-cts-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG { 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG "data": { 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG }, 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "value": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG }, 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 DEBUG 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG [...] 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG }, 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG "0" 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG ] 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG } 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG ] 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG } 00:42:46 DEBUG "2125.3333333333335" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG ] 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "instance": "10.142.0.107:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "instance": "10.142.0.43:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0.00040000000000001514" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-2" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "5360.177169317494" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0.011" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "2107.366666666667" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "5.923880077791209" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-cts-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0.000159625" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-2" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "379426.5" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "2107.1666666666665" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "instance": "10.142.0.107:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "instance": "10.142.0.43:9100" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "12184.33333333333" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "ds-idrepo-1" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051766" 00:42:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051766" 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "pod": "ds-idrepo-0" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "0" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:46 INFO [http_cmd]: http status code OK 00:42:46 DEBUG --- status code --- 00:42:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:42:46 DEBUG --- http response --- 00:42:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:42:46 DEBUG ----- output ----- 00:42:46 DEBUG { 00:42:46 DEBUG "status": "success", 00:42:46 DEBUG "data": { 00:42:46 DEBUG "resultType": "vector", 00:42:46 DEBUG "result": [ 00:42:46 DEBUG { 00:42:46 DEBUG "metric": { 00:42:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG [...] 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG 00:42:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:42:46 DEBUG }, 00:42:46 DEBUG "value": [ 00:42:46 DEBUG 1713051766, 00:42:46 DEBUG "1823109120" 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG ] 00:42:46 DEBUG } 00:42:46 DEBUG } 00:42:47 INFO 00:42:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:42:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:42:47 INFO [loop_until]: OK (rc = 0) 00:42:47 DEBUG --- stdout --- 00:42:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11150m 8990Mi am-7d597cf7d6-8vr8f 9910m 8381Mi am-7d597cf7d6-9wz9h 13520m 9112Mi ds-cts-0 10171m 7137Mi ds-cts-1 10358m 7574Mi ds-cts-2 10501m 7313Mi ds-idrepo-0 11951m 4268Mi ds-idrepo-1 1785m 4290Mi ds-idrepo-2 1712m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5952m 1646Mi 00:42:47 DEBUG --- stderr --- 00:42:47 DEBUG 00:42:48 INFO 00:42:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:42:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:42:48 INFO [loop_until]: OK (rc = 0) 00:42:48 DEBUG --- stdout --- 00:42:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4813Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10534m 35% 9390Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11582m 38% 11045Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 15573m 52% 2985Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1877m 6% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10321m 34% 10553Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 61m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10974m 36% 9803Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1813m 6% 5447Mi 4% gke-xlou-cdm-ds-0f601751-j488 10519m 35% 8960Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2212Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14874m 49% 10645Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11985m 40% 5660Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6038m 37% 5390Mi 9% 00:42:48 DEBUG --- stderr --- 00:42:48 DEBUG 00:43:42 WARNING File /tmp/lodemon_data-20240414_000936/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [14/Apr/2024 00:43:42] "GET /monitoring/average?start_time=20240413_231433&stop_time=20240413_234141 HTTP/1.1" 200 - 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "2105.866666666667" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 ERROR [http_cmd]: ERROR 00:43:46 ERROR --- status code --- 00:43:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:43:46 ERROR --- http response --- 00:43:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "instance": "10.142.0.107:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "instance": "10.142.0.43:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "12584410112" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "instance": "10.142.0.107:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "instance": "10.142.0.43:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "ds-idrepo-0" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "instance": "10.142.0.107:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "instance": "10.142.0.43:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "6853.8" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG { 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "ds-cts-1" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "instance": "10.142.0.107:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "instance": "10.142.0.43:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0.000500000000000019" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "2113.9333333333334" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "ds-idrepo-0" 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG }, 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG "0.011" 00:43:46 DEBUG ] 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "500109.8571003999" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "5.486844893016923" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG "result": [ 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG "result": [ 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG { 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG "data": { 00:43:46 DEBUG }, 00:43:46 DEBUG }, 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "value": [ 00:43:46 DEBUG "value": [ 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG "result": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG { 00:43:46 DEBUG { 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "metric": { 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "data": { 00:43:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:43:46 DEBUG [...] 00:43:46 DEBUG [...] 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG }, 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "result": [ 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG { 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "metric": { 00:43:46 DEBUG 00:43:46 DEBUG "pod": "ds-idrepo-0" 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG 00:43:46 DEBUG }, 00:43:46 DEBUG }, 00:43:46 DEBUG }, 00:43:46 DEBUG [...] 00:43:46 DEBUG "value": [ 00:43:46 DEBUG "value": [ 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "0" 00:43:46 DEBUG "0" 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG ] 00:43:46 DEBUG ] 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 DEBUG [...] 00:43:46 DEBUG }, 00:43:46 DEBUG ] 00:43:46 DEBUG ] 00:43:46 DEBUG 00:43:46 DEBUG "value": [ 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 DEBUG 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 DEBUG 00:43:46 DEBUG "0" 00:43:46 DEBUG "pod": "ds-cts-1" 00:43:46 DEBUG ] 00:43:46 DEBUG }, 00:43:46 DEBUG } 00:43:46 DEBUG "value": [ 00:43:46 DEBUG ] 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG } 00:43:46 DEBUG "0.00003975" 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "ds-idrepo-0" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "2105.6" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "2106.8" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "378904" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "ds-idrepo-1" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "ds-idrepo-0" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "instance": "10.142.0.107:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "instance": "10.142.0.43:9100" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "12354.499999999998" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051826" 00:43:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051826" 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "0" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 INFO [http_cmd]: http status code OK 00:43:46 DEBUG } 00:43:46 DEBUG --- status code --- 00:43:46 DEBUG } 00:43:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:43:46 DEBUG --- http response --- 00:43:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:43:46 DEBUG ----- output ----- 00:43:46 DEBUG { 00:43:46 DEBUG "status": "success", 00:43:46 DEBUG "data": { 00:43:46 DEBUG "resultType": "vector", 00:43:46 DEBUG "result": [ 00:43:46 DEBUG { 00:43:46 DEBUG "metric": { 00:43:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG [...] 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG 00:43:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:43:46 DEBUG }, 00:43:46 DEBUG "value": [ 00:43:46 DEBUG 1713051826, 00:43:46 DEBUG "1204060160" 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG ] 00:43:46 DEBUG } 00:43:46 DEBUG } 00:43:47 INFO 00:43:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:43:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:43:47 INFO [loop_until]: OK (rc = 0) 00:43:47 DEBUG --- stdout --- 00:43:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10320m 8997Mi am-7d597cf7d6-8vr8f 9292m 8388Mi am-7d597cf7d6-9wz9h 14139m 9115Mi ds-cts-0 9781m 7135Mi ds-cts-1 10201m 7566Mi ds-cts-2 10876m 7316Mi ds-idrepo-0 11591m 4269Mi ds-idrepo-1 1734m 4290Mi ds-idrepo-2 1673m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 963Mi idm-b85f9d978-rl76f 9m 968Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 5338m 1057Mi 00:43:47 DEBUG --- stderr --- 00:43:47 DEBUG 00:43:48 INFO 00:43:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:43:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:43:48 INFO [loop_until]: OK (rc = 0) 00:43:48 DEBUG --- stdout --- 00:43:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4802Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10481m 35% 9409Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 11625m 38% 11059Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 13163m 44% 2981Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1673m 5% 6274Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10210m 34% 10552Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 60m 0% 2890Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11109m 37% 9808Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1568m 5% 5438Mi 4% gke-xlou-cdm-ds-0f601751-j488 10610m 35% 8966Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2220Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 14743m 49% 10649Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11903m 39% 5664Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 5856m 36% 4807Mi 8% 00:43:48 DEBUG --- stderr --- 00:43:48 DEBUG 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "instance": "10.142.0.107:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "instance": "10.142.0.43:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "12584954880" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051886" 00:44:46 ERROR [http_cmd]: ERROR 00:44:46 ERROR --- status code --- 00:44:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:44:46 ERROR --- http response --- 00:44:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:44:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "382.1666666666667" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "instance": "10.142.0.107:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "instance": "10.142.0.43:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "ds-idrepo-0" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "ds-cts-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "instance": "10.142.0.107:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "instance": "10.142.0.43:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "5312.233333333333" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "1099531.9337857552" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "3.3655871405445685" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "ds-cts-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "instance": "10.142.0.107:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "instance": "10.142.0.43:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0.00040000000000001514" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "127683.38935751888" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "2092.5333333333333" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "ds-idrepo-0" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0.011" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "ds-idrepo-0" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG 00:44:46 DEBUG { 00:44:46 DEBUG 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG [...] 00:44:46 DEBUG "data": { 00:44:46 DEBUG 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG 00:44:46 DEBUG "result": [ 00:44:46 DEBUG 00:44:46 DEBUG { 00:44:46 DEBUG "pod": "ds-idrepo-0" 00:44:46 DEBUG "metric": { 00:44:46 DEBUG }, 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG "value": [ 00:44:46 DEBUG }, 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG "0" 00:44:46 DEBUG 00:44:46 DEBUG ] 00:44:46 DEBUG 00:44:46 DEBUG } 00:44:46 DEBUG 00:44:46 DEBUG ] 00:44:46 DEBUG [...] 00:44:46 DEBUG } 00:44:46 DEBUG 00:44:46 DEBUG } 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "381.3333333333333" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "381.96666666666664" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "296850.5" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "ds-idrepo-1" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "ds-idrepo-0" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "0" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "instance": "10.142.0.107:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "instance": "10.142.0.43:9100" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "11732.266666666665" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051886" 00:44:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051886" 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "3.5704628971903296" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:46 INFO [http_cmd]: http status code OK 00:44:46 DEBUG --- status code --- 00:44:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:44:46 DEBUG --- http response --- 00:44:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:44:46 DEBUG ----- output ----- 00:44:46 DEBUG { 00:44:46 DEBUG "status": "success", 00:44:46 DEBUG "data": { 00:44:46 DEBUG "resultType": "vector", 00:44:46 DEBUG "result": [ 00:44:46 DEBUG { 00:44:46 DEBUG "metric": { 00:44:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG [...] 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG 00:44:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:44:46 DEBUG }, 00:44:46 DEBUG "value": [ 00:44:46 DEBUG 1713051886, 00:44:46 DEBUG "1677889536" 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG ] 00:44:46 DEBUG } 00:44:46 DEBUG } 00:44:47 INFO 00:44:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:44:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:44:47 INFO [loop_until]: OK (rc = 0) 00:44:47 DEBUG --- stdout --- 00:44:47 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 2981m 8990Mi am-7d597cf7d6-8vr8f 3485m 8388Mi am-7d597cf7d6-9wz9h 3549m 8203Mi ds-cts-0 6041m 7157Mi ds-cts-1 6794m 7569Mi ds-cts-2 5446m 7373Mi ds-idrepo-0 2405m 4268Mi ds-idrepo-1 536m 4291Mi ds-idrepo-2 580m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 2556m 1495Mi 00:44:47 DEBUG --- stderr --- 00:44:47 DEBUG 00:44:48 INFO 00:44:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:44:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:44:48 INFO [loop_until]: OK (rc = 0) 00:44:48 DEBUG --- stdout --- 00:44:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4804Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5761m 19% 9458Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 3323m 11% 11076Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 5538m 18% 2969Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 836m 2% 6287Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 5846m 19% 10557Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2894Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 3379m 11% 9815Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 762m 2% 5453Mi 4% gke-xlou-cdm-ds-0f601751-j488 6328m 21% 8971Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2209Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 4590m 15% 9749Mi 8% gke-xlou-cdm-ds-0f601751-nmxm 3247m 10% 5663Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 3295m 20% 5297Mi 9% 00:44:48 DEBUG --- stderr --- 00:44:48 DEBUG 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713051946" 00:45:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "instance": "10.142.0.107:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "instance": "10.142.0.43:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "12585561088" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "instance": "10.142.0.107:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "instance": "10.142.0.43:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0.00033333333333333365" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 ERROR [http_cmd]: ERROR 00:45:46 ERROR --- status code --- 00:45:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:45:46 ERROR --- http response --- 00:45:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:45:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "2293.1666666666665" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "instance": "10.142.0.107:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "instance": "10.142.0.43:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "5256.933333333333" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:45:46 DEBUG { 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG { 00:45:46 DEBUG "data": { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "result": [ 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG { 00:45:46 DEBUG "result": [ 00:45:46 DEBUG "metric": { 00:45:46 DEBUG { 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG "metric": { 00:45:46 DEBUG }, 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG "value": [ 00:45:46 DEBUG }, 00:45:46 DEBUG 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG 00:45:46 DEBUG }, 00:45:46 DEBUG "pod": "ds-cts-1" 00:45:46 DEBUG "value": [ 00:45:46 DEBUG }, 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG "0" 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG ] 00:45:46 DEBUG "0" 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "instance": "10.142.0.107:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "instance": "10.142.0.43:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0.0004999999999997821" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "151131188.4977556" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "4838.502818184887" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "6.610597294331141" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-cts-1" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0.0002" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "2107.7" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG { 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG 00:45:46 DEBUG { 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG }, 00:45:46 DEBUG "data": { 00:45:46 DEBUG "value": [ 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "result": [ 00:45:46 DEBUG "0" 00:45:46 DEBUG { 00:45:46 DEBUG ] 00:45:46 DEBUG "metric": { 00:45:46 DEBUG } 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG ] 00:45:46 DEBUG }, 00:45:46 DEBUG } 00:45:46 DEBUG "value": [ 00:45:46 DEBUG } 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0.011" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "2288.633333333333" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "2288.733333333333" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "273260" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "ds-idrepo-1" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "0" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "instance": "10.142.0.107:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "instance": "10.142.0.43:9100" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "11691.133333333331" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713051946" 00:45:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713051946" 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "pod": "ds-idrepo-0" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "13.01936815282324" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:46 INFO [http_cmd]: http status code OK 00:45:46 DEBUG --- status code --- 00:45:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:45:46 DEBUG --- http response --- 00:45:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:45:46 DEBUG ----- output ----- 00:45:46 DEBUG { 00:45:46 DEBUG "status": "success", 00:45:46 DEBUG "data": { 00:45:46 DEBUG "resultType": "vector", 00:45:46 DEBUG "result": [ 00:45:46 DEBUG { 00:45:46 DEBUG "metric": { 00:45:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG [...] 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG 00:45:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:45:46 DEBUG }, 00:45:46 DEBUG "value": [ 00:45:46 DEBUG 1713051946, 00:45:46 DEBUG "1736880128" 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG ] 00:45:46 DEBUG } 00:45:46 DEBUG } 00:45:47 INFO 00:45:47 INFO [loop_until]: kubectl --namespace=xlou top pods 00:45:47 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:45:48 INFO [loop_until]: OK (rc = 0) 00:45:48 DEBUG --- stdout --- 00:45:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 11830m 8993Mi am-7d597cf7d6-8vr8f 11187m 8378Mi am-7d597cf7d6-9wz9h 14844m 8208Mi ds-cts-0 10702m 7119Mi ds-cts-1 10694m 7561Mi ds-cts-2 10744m 7366Mi ds-idrepo-0 13056m 4268Mi ds-idrepo-1 1938m 4291Mi ds-idrepo-2 1836m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6616m 1563Mi 00:45:48 DEBUG --- stderr --- 00:45:48 DEBUG 00:45:48 INFO 00:45:48 INFO [loop_until]: kubectl --namespace=xlou top node 00:45:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:45:48 INFO [loop_until]: OK (rc = 0) 00:45:48 DEBUG --- stdout --- 00:45:48 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4817Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10719m 35% 9465Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12549m 42% 11053Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17426m 58% 2994Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2011m 6% 6287Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10411m 34% 10557Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2902Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11925m 39% 9804Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1942m 6% 5444Mi 4% gke-xlou-cdm-ds-0f601751-j488 10842m 36% 8973Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2207Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16029m 53% 9739Mi 8% gke-xlou-cdm-ds-0f601751-nmxm 13034m 43% 5666Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6757m 42% 5347Mi 9% 00:45:48 DEBUG --- stderr --- 00:45:48 DEBUG 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052006" 00:46:46 ERROR [http_cmd]: ERROR 00:46:46 ERROR --- status code --- 00:46:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:46:46 ERROR --- http response --- 00:46:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:46:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "instance": "10.142.0.107:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "instance": "10.142.0.43:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0.00033333333333333365" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "instance": "10.142.0.107:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "instance": "10.142.0.43:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "12586177536" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "2299.9" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-cts-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "instance": "10.142.0.107:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "instance": "10.142.0.43:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0.0004333333333332708" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "instance": "10.142.0.107:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "instance": "10.142.0.43:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "5374.2" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0.011" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-idrepo-0" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "2299.5333333333333" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "158703.44278024934" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-cts-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0.000033333333333333335" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "6.583074818052989" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "2108.0666666666666" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "data": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG }, 00:46:46 DEBUG "result": [ 00:46:46 DEBUG "value": [ 00:46:46 DEBUG { 00:46:46 DEBUG 00:46:46 DEBUG "metric": { 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG 00:46:46 DEBUG }, 00:46:46 DEBUG [...] 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-idrepo-0" 00:46:46 DEBUG [...] 00:46:46 DEBUG }, 00:46:46 DEBUG 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG } 00:46:46 DEBUG }, 00:46:46 DEBUG ] 00:46:46 DEBUG "value": [ 00:46:46 DEBUG } 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG } 00:46:46 DEBUG "2298.633333333333" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-idrepo-0" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-idrepo-0" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "283913.5" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "1741565952" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "13.094876644184996" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052006" 00:46:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052006" 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "instance": "10.142.0.107:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "instance": "10.142.0.43:9100" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "11869.766666666665" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:46 INFO [http_cmd]: http status code OK 00:46:46 DEBUG --- status code --- 00:46:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:46:46 DEBUG --- http response --- 00:46:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:46:46 DEBUG ----- output ----- 00:46:46 DEBUG { 00:46:46 DEBUG "status": "success", 00:46:46 DEBUG "data": { 00:46:46 DEBUG "resultType": "vector", 00:46:46 DEBUG "result": [ 00:46:46 DEBUG { 00:46:46 DEBUG "metric": { 00:46:46 DEBUG "pod": "ds-idrepo-1" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG [...] 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG 00:46:46 DEBUG "pod": "ds-idrepo-0" 00:46:46 DEBUG }, 00:46:46 DEBUG "value": [ 00:46:46 DEBUG 1713052006, 00:46:46 DEBUG "0" 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG ] 00:46:46 DEBUG } 00:46:46 DEBUG } 00:46:48 INFO 00:46:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:46:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:46:48 INFO [loop_until]: OK (rc = 0) 00:46:48 DEBUG --- stdout --- 00:46:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12170m 8988Mi am-7d597cf7d6-8vr8f 10984m 8376Mi am-7d597cf7d6-9wz9h 15771m 8210Mi ds-cts-0 10308m 7145Mi ds-cts-1 10797m 7586Mi ds-cts-2 10951m 7365Mi ds-idrepo-0 12976m 4268Mi ds-idrepo-1 1942m 4291Mi ds-idrepo-2 1855m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6628m 1567Mi 00:46:48 DEBUG --- stderr --- 00:46:48 DEBUG 00:46:49 INFO 00:46:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:46:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:46:49 INFO [loop_until]: OK (rc = 0) 00:46:49 DEBUG --- stdout --- 00:46:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4809Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 10757m 36% 9463Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12656m 42% 11047Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17422m 58% 2991Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2037m 6% 6295Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10424m 34% 10562Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12013m 40% 9802Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1943m 6% 5441Mi 4% gke-xlou-cdm-ds-0f601751-j488 10880m 36% 8988Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2225Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16433m 55% 9738Mi 8% gke-xlou-cdm-ds-0f601751-nmxm 13060m 43% 5662Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6736m 42% 5359Mi 9% 00:46:49 DEBUG --- stderr --- 00:46:49 DEBUG 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052066" 00:47:46 ERROR [http_cmd]: ERROR 00:47:46 ERROR --- status code --- 00:47:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:47:46 ERROR --- http response --- 00:47:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:47:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "instance": "10.142.0.107:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "instance": "10.142.0.43:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "2300.9" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "instance": "10.142.0.107:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "instance": "10.142.0.43:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "12586785792" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "ds-idrepo-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "ds-cts-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "instance": "10.142.0.107:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "instance": "10.142.0.43:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0.0004666666666667633" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "instance": "10.142.0.107:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "instance": "10.142.0.43:9100" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "5181" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG { 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG "data": { 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG }, 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "value": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "ds-idrepo-1" 00:47:46 DEBUG 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG }, 00:47:46 DEBUG 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG [...] 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG }, 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG 00:47:46 DEBUG "pod": "ds-idrepo-0" 00:47:46 DEBUG "0.011" 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG }, 00:47:46 DEBUG ] 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG } 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG ] 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG } 00:47:46 DEBUG "2300.866666666667" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "379.7" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG { 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:47:46 DEBUG "result": [ 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG "data": { 00:47:46 DEBUG }, 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "value": [ 00:47:46 DEBUG "result": [ 00:47:46 DEBUG 00:47:46 DEBUG { 00:47:46 DEBUG 00:47:46 DEBUG "metric": { 00:47:46 DEBUG 00:47:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:47:46 DEBUG [...] 00:47:46 DEBUG }, 00:47:46 DEBUG 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG 00:47:46 DEBUG }, 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG [...] 00:47:46 DEBUG "value": [ 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG 00:47:46 DEBUG "2300.866666666667" 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG 00:47:46 DEBUG ] 00:47:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:47:46 DEBUG } 00:47:46 DEBUG }, 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG ] 00:47:46 DEBUG "value": [ 00:47:46 DEBUG { 00:47:46 DEBUG } 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG } 00:47:46 DEBUG "82316.6658309041" 00:47:46 DEBUG "data": { 00:47:46 DEBUG ] 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG } 00:47:46 DEBUG "result": [ 00:47:46 DEBUG ] 00:47:46 DEBUG { 00:47:46 DEBUG } 00:47:46 DEBUG "metric": { 00:47:46 DEBUG } 00:47:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "ds-idrepo-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "ds-cts-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0.00023208333333333333" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "6.542338955879789" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "ds-idrepo-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "ds-idrepo-0" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "ds-idrepo-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "ds-idrepo-0" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "ds-idrepo-1" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "ds-idrepo-0" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "377666" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "0" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:46 INFO [http_cmd]: http status code OK 00:47:46 DEBUG --- status code --- 00:47:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:46 DEBUG --- http response --- 00:47:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:47:46 DEBUG ----- output ----- 00:47:46 DEBUG { 00:47:46 DEBUG "status": "success", 00:47:46 DEBUG "data": { 00:47:46 DEBUG "resultType": "vector", 00:47:46 DEBUG "result": [ 00:47:46 DEBUG { 00:47:46 DEBUG "metric": { 00:47:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG [...] 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG 00:47:46 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:47:46 DEBUG }, 00:47:46 DEBUG "value": [ 00:47:46 DEBUG 1713052066, 00:47:46 DEBUG "10.818313854288956" 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG ] 00:47:46 DEBUG } 00:47:46 DEBUG } 00:47:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052066" 00:47:47 INFO [http_cmd]: http status code OK 00:47:47 DEBUG --- status code --- 00:47:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:47 DEBUG --- http response --- 00:47:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:47:47 DEBUG ----- output ----- 00:47:47 DEBUG { 00:47:47 DEBUG "status": "success", 00:47:47 DEBUG "data": { 00:47:47 DEBUG "resultType": "vector", 00:47:47 DEBUG "result": [ 00:47:47 DEBUG { 00:47:47 DEBUG "metric": { 00:47:47 DEBUG "pod": "ds-idrepo-1" 00:47:47 DEBUG }, 00:47:47 DEBUG "value": [ 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG [...] 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG "pod": "ds-idrepo-0" 00:47:47 DEBUG }, 00:47:47 DEBUG "value": [ 00:47:47 DEBUG 1713052066, 00:47:47 DEBUG "0" 00:47:47 DEBUG ] 00:47:47 DEBUG } 00:47:47 DEBUG ] 00:47:47 DEBUG } 00:47:47 DEBUG } 00:47:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052067" 00:47:47 INFO [http_cmd]: http status code OK 00:47:47 DEBUG --- status code --- 00:47:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:47 DEBUG --- http response --- 00:47:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:47 DEBUG ----- output ----- 00:47:47 DEBUG { 00:47:47 DEBUG "status": "success", 00:47:47 DEBUG "data": { 00:47:47 DEBUG "resultType": "vector", 00:47:47 DEBUG "result": [ 00:47:47 DEBUG { 00:47:47 DEBUG "metric": { 00:47:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:47:47 DEBUG }, 00:47:47 DEBUG "value": [ 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG [...] 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:47:47 DEBUG }, 00:47:47 DEBUG "value": [ 00:47:47 DEBUG 1713052067, 00:47:47 DEBUG "1744347136" 00:47:47 DEBUG ] 00:47:47 DEBUG } 00:47:47 DEBUG ] 00:47:47 DEBUG } 00:47:47 DEBUG } 00:47:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052067" 00:47:47 INFO [http_cmd]: http status code OK 00:47:47 DEBUG --- status code --- 00:47:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:47:47 DEBUG --- http response --- 00:47:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:47:47 DEBUG ----- output ----- 00:47:47 DEBUG { 00:47:47 DEBUG "status": "success", 00:47:47 DEBUG "data": { 00:47:47 DEBUG "resultType": "vector", 00:47:47 DEBUG "result": [ 00:47:47 DEBUG { 00:47:47 DEBUG "metric": { 00:47:47 DEBUG "instance": "10.142.0.107:9100" 00:47:47 DEBUG }, 00:47:47 DEBUG "value": [ 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG [...] 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG 00:47:47 DEBUG "instance": "10.142.0.43:9100" 00:47:47 DEBUG }, 00:47:47 DEBUG "value": [ 00:47:47 DEBUG 1713052067, 00:47:47 DEBUG "11681.966666666664" 00:47:47 DEBUG ] 00:47:47 DEBUG } 00:47:47 DEBUG ] 00:47:47 DEBUG } 00:47:47 DEBUG } 00:47:48 INFO 00:47:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:47:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:47:48 INFO [loop_until]: OK (rc = 0) 00:47:48 DEBUG --- stdout --- 00:47:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 10749m 8984Mi am-7d597cf7d6-8vr8f 9256m 8377Mi am-7d597cf7d6-9wz9h 12619m 9076Mi ds-cts-0 5321m 7169Mi ds-cts-1 6134m 7598Mi ds-cts-2 5483m 7389Mi ds-idrepo-0 11705m 4268Mi ds-idrepo-1 1849m 4291Mi ds-idrepo-2 1734m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6468m 1570Mi 00:47:48 DEBUG --- stderr --- 00:47:48 DEBUG 00:47:49 INFO 00:47:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:47:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:47:49 INFO [loop_until]: OK (rc = 0) 00:47:49 DEBUG --- stdout --- 00:47:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4814Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 6024m 20% 9507Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 10588m 35% 11047Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17386m 58% 2990Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1967m 6% 6291Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 5833m 19% 10607Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2894Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 10009m 33% 9798Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1866m 6% 5449Mi 4% gke-xlou-cdm-ds-0f601751-j488 6459m 21% 9017Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2213Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 13686m 45% 10615Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 11781m 39% 5652Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6660m 41% 5353Mi 9% 00:47:49 DEBUG --- stderr --- 00:47:49 DEBUG 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052126" 00:48:46 ERROR [http_cmd]: ERROR 00:48:46 ERROR --- status code --- 00:48:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:48:46 ERROR --- http response --- 00:48:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:48:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "instance": "10.142.0.107:9100" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "instance": "10.142.0.43:9100" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0.000333333333333341" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "2291.2" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "instance": "10.142.0.107:9100" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "instance": "10.142.0.43:9100" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "12587609088" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0.011" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "2292.133333333333" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "ds-idrepo-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-cts-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "instance": "10.142.0.107:9100" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "instance": "10.142.0.43:9100" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0.0004333333333332708" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG { 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG "metric": { 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG "instance": "10.142.0.107:9100" 00:48:46 DEBUG { 00:48:46 DEBUG }, 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "value": [ 00:48:46 DEBUG "data": { 00:48:46 DEBUG 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG 00:48:46 DEBUG "result": [ 00:48:46 DEBUG 00:48:46 DEBUG { 00:48:46 DEBUG [...] 00:48:46 DEBUG "metric": { 00:48:46 DEBUG 00:48:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:46 DEBUG 00:48:46 DEBUG }, 00:48:46 DEBUG 00:48:46 DEBUG "value": [ 00:48:46 DEBUG "instance": "10.142.0.43:9100" 00:48:46 DEBUG 00:48:46 DEBUG }, 00:48:46 DEBUG 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG [...] 00:48:46 DEBUG "6748.766666666666" 00:48:46 DEBUG 00:48:46 DEBUG ] 00:48:46 DEBUG 00:48:46 DEBUG } 00:48:46 DEBUG 00:48:46 DEBUG ] 00:48:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:46 DEBUG } 00:48:46 DEBUG }, 00:48:46 DEBUG } 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "2291.633333333333" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "data": { 00:48:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG }, 00:48:46 DEBUG "result": [ 00:48:46 DEBUG "value": [ 00:48:46 DEBUG { 00:48:46 DEBUG 00:48:46 DEBUG "metric": { 00:48:46 DEBUG 00:48:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:46 DEBUG 00:48:46 DEBUG }, 00:48:46 DEBUG [...] 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:46 DEBUG [...] 00:48:46 DEBUG }, 00:48:46 DEBUG 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG 00:48:46 DEBUG "2291.7" 00:48:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:46 DEBUG ] 00:48:46 DEBUG }, 00:48:46 DEBUG } 00:48:46 DEBUG "value": [ 00:48:46 DEBUG ] 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG } 00:48:46 DEBUG "0" 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "ds-idrepo-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-idrepo-0" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "ds-idrepo-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-cts-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:48:46 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:48:46 DEBUG }, 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-idrepo-0" 00:48:46 DEBUG "pod": "idm-b85f9d978-rl76f" 00:48:46 DEBUG }, 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG "410.45291974078657" 00:48:46 DEBUG ] 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "6.537164516455761" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "ds-idrepo-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-idrepo-0" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "ds-idrepo-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-idrepo-0" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052126" 00:48:46 INFO [http_cmd]: http status code OK 00:48:46 DEBUG --- status code --- 00:48:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:46 DEBUG --- http response --- 00:48:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:48:46 DEBUG ----- output ----- 00:48:46 DEBUG { 00:48:46 DEBUG "status": "success", 00:48:46 DEBUG "data": { 00:48:46 DEBUG "resultType": "vector", 00:48:46 DEBUG "result": [ 00:48:46 DEBUG { 00:48:46 DEBUG "metric": { 00:48:46 DEBUG "pod": "ds-idrepo-1" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG [...] 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG 00:48:46 DEBUG "pod": "ds-idrepo-0" 00:48:46 DEBUG }, 00:48:46 DEBUG "value": [ 00:48:46 DEBUG 1713052126, 00:48:46 DEBUG "0" 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG ] 00:48:46 DEBUG } 00:48:46 DEBUG } 00:48:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052127" 00:48:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052127" 00:48:47 INFO [http_cmd]: http status code OK 00:48:47 DEBUG --- status code --- 00:48:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:47 DEBUG --- http response --- 00:48:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:47 DEBUG ----- output ----- 00:48:47 DEBUG { 00:48:47 DEBUG "status": "success", 00:48:47 DEBUG "data": { 00:48:47 DEBUG "resultType": "vector", 00:48:47 DEBUG "result": [ 00:48:47 DEBUG { 00:48:47 DEBUG "metric": { 00:48:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG [...] 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG "pod": "ds-cts-2" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 1713052127, 00:48:47 DEBUG "29983551.505037293" 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG } 00:48:47 INFO [http_cmd]: http status code OK 00:48:47 DEBUG --- status code --- 00:48:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:47 DEBUG --- http response --- 00:48:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:47 DEBUG ----- output ----- 00:48:47 DEBUG { 00:48:47 DEBUG "status": "success", 00:48:47 DEBUG "data": { 00:48:47 DEBUG "resultType": "vector", 00:48:47 DEBUG "result": [ 00:48:47 DEBUG { 00:48:47 DEBUG "metric": { 00:48:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG [...] 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 1713052127, 00:48:47 DEBUG "413090.5" 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG } 00:48:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052127" 00:48:47 INFO [http_cmd]: http status code OK 00:48:47 DEBUG --- status code --- 00:48:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:47 DEBUG --- http response --- 00:48:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:48:47 DEBUG ----- output ----- 00:48:47 DEBUG { 00:48:47 DEBUG "status": "success", 00:48:47 DEBUG "data": { 00:48:47 DEBUG "resultType": "vector", 00:48:47 DEBUG "result": [ 00:48:47 DEBUG { 00:48:47 DEBUG "metric": { 00:48:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG [...] 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG "pod": "ds-idrepo-0" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 1713052127, 00:48:47 DEBUG "0" 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG } 00:48:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052127" 00:48:47 INFO [http_cmd]: http status code OK 00:48:47 DEBUG --- status code --- 00:48:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:47 DEBUG --- http response --- 00:48:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:47 DEBUG ----- output ----- 00:48:47 DEBUG { 00:48:47 DEBUG "status": "success", 00:48:47 DEBUG "data": { 00:48:47 DEBUG "resultType": "vector", 00:48:47 DEBUG "result": [ 00:48:47 DEBUG { 00:48:47 DEBUG "metric": { 00:48:47 DEBUG "instance": "10.142.0.107:9100" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG [...] 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG "instance": "10.142.0.43:9100" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 1713052127, 00:48:47 DEBUG "12217.899999999998" 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG } 00:48:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052127" 00:48:47 INFO [http_cmd]: http status code OK 00:48:47 DEBUG --- status code --- 00:48:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:47 DEBUG --- http response --- 00:48:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:48:47 DEBUG ----- output ----- 00:48:47 DEBUG { 00:48:47 DEBUG "status": "success", 00:48:47 DEBUG "data": { 00:48:47 DEBUG "resultType": "vector", 00:48:47 DEBUG "result": [ 00:48:47 DEBUG { 00:48:47 DEBUG "metric": { 00:48:47 DEBUG "pod": "ds-idrepo-1" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG [...] 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG "pod": "ds-idrepo-0" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 1713052127, 00:48:47 DEBUG "0" 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG } 00:48:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052127" 00:48:47 INFO [http_cmd]: http status code OK 00:48:47 DEBUG --- status code --- 00:48:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:48:47 DEBUG --- http response --- 00:48:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:48:47 DEBUG ----- output ----- 00:48:47 DEBUG { 00:48:47 DEBUG "status": "success", 00:48:47 DEBUG "data": { 00:48:47 DEBUG "resultType": "vector", 00:48:47 DEBUG "result": [ 00:48:47 DEBUG { 00:48:47 DEBUG "metric": { 00:48:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG [...] 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG 00:48:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:48:47 DEBUG }, 00:48:47 DEBUG "value": [ 00:48:47 DEBUG 1713052127, 00:48:47 DEBUG "1746804736" 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG ] 00:48:47 DEBUG } 00:48:47 DEBUG } 00:48:48 INFO 00:48:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:48:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:48:48 INFO [loop_until]: OK (rc = 0) 00:48:48 DEBUG --- stdout --- 00:48:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12531m 8992Mi am-7d597cf7d6-8vr8f 11189m 8377Mi am-7d597cf7d6-9wz9h 15285m 9090Mi ds-cts-0 11230m 7337Mi ds-cts-1 11524m 7649Mi ds-cts-2 11643m 7429Mi ds-idrepo-0 13025m 4268Mi ds-idrepo-1 1962m 4290Mi ds-idrepo-2 1864m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 6m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6308m 1571Mi 00:48:48 DEBUG --- stderr --- 00:48:48 DEBUG 00:48:49 INFO 00:48:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:48:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:48:49 INFO [loop_until]: OK (rc = 0) 00:48:49 DEBUG --- stdout --- 00:48:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4811Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11194m 37% 9532Mi 8% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12827m 42% 11048Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17422m 58% 2994Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2027m 6% 6281Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11404m 38% 10747Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2896Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12237m 40% 9809Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1943m 6% 5444Mi 4% gke-xlou-cdm-ds-0f601751-j488 11570m 38% 9042Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2216Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16318m 54% 10623Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13181m 44% 5658Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6671m 41% 5354Mi 9% 00:48:49 DEBUG --- stderr --- 00:48:49 DEBUG 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 ERROR [http_cmd]: ERROR 00:49:46 ERROR --- status code --- 00:49:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:49:46 ERROR --- http response --- 00:49:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:49:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "instance": "10.142.0.107:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "instance": "10.142.0.43:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "2303.766666666667" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "instance": "10.142.0.107:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "instance": "10.142.0.43:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "12588243968" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "instance": "10.142.0.107:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "instance": "10.142.0.43:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0.0004666666666667633" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0.011" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "2308.866666666667" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "ds-idrepo-1" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "ds-cts-1" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "ds-idrepo-1" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "ds-cts-1" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "2309.2666666666664" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "instance": "10.142.0.107:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "instance": "10.142.0.43:9100" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "5319.933333333333" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "2298.5333333333333" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "ds-idrepo-1" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "ds-idrepo-0" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052186" 00:49:46 INFO [http_cmd]: http status code OK 00:49:46 DEBUG --- status code --- 00:49:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:46 DEBUG --- http response --- 00:49:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:49:46 DEBUG ----- output ----- 00:49:46 DEBUG { 00:49:46 DEBUG "status": "success", 00:49:46 DEBUG "data": { 00:49:46 DEBUG "resultType": "vector", 00:49:46 DEBUG "result": [ 00:49:46 DEBUG { 00:49:46 DEBUG "metric": { 00:49:46 DEBUG "pod": "ds-idrepo-1" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG [...] 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG 00:49:46 DEBUG "pod": "ds-idrepo-0" 00:49:46 DEBUG }, 00:49:46 DEBUG "value": [ 00:49:46 DEBUG 1713052186, 00:49:46 DEBUG "0" 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG ] 00:49:46 DEBUG } 00:49:46 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "ds-idrepo-1" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "ds-idrepo-0" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052186, 00:49:47 DEBUG "0" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052187" 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "241216413.1731064" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "4236213.973203648" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "6.556430567123923" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "414046" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "ds-idrepo-1" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "ds-idrepo-0" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "0" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052187" 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "0" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "instance": "10.142.0.107:9100" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "instance": "10.142.0.43:9100" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "12111.066666666664" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "ds-idrepo-1" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "ds-idrepo-0" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "0" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052187" 00:49:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052187" 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "7.842265642869299" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:47 INFO [http_cmd]: http status code OK 00:49:47 DEBUG --- status code --- 00:49:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:49:47 DEBUG --- http response --- 00:49:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:49:47 DEBUG ----- output ----- 00:49:47 DEBUG { 00:49:47 DEBUG "status": "success", 00:49:47 DEBUG "data": { 00:49:47 DEBUG "resultType": "vector", 00:49:47 DEBUG "result": [ 00:49:47 DEBUG { 00:49:47 DEBUG "metric": { 00:49:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG [...] 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG 00:49:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:49:47 DEBUG }, 00:49:47 DEBUG "value": [ 00:49:47 DEBUG 1713052187, 00:49:47 DEBUG "1781235712" 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG ] 00:49:47 DEBUG } 00:49:47 DEBUG } 00:49:48 INFO 00:49:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:49:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:49:48 INFO [loop_until]: OK (rc = 0) 00:49:48 DEBUG --- stdout --- 00:49:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12601m 8990Mi am-7d597cf7d6-8vr8f 11344m 8389Mi am-7d597cf7d6-9wz9h 15404m 9104Mi ds-cts-0 11287m 7512Mi ds-cts-1 11510m 7743Mi ds-cts-2 11378m 8001Mi ds-idrepo-0 13416m 4269Mi ds-idrepo-1 2002m 4291Mi ds-idrepo-2 1820m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 963Mi idm-b85f9d978-rl76f 4m 968Mi lodemon-64cbdcc7b4-vt4kk 6m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6536m 1605Mi 00:49:48 DEBUG --- stderr --- 00:49:48 DEBUG 00:49:49 INFO 00:49:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:49:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:49:49 INFO [loop_until]: OK (rc = 0) 00:49:49 DEBUG --- stdout --- 00:49:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4803Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11454m 38% 10117Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12795m 42% 11047Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17470m 58% 3006Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2056m 6% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11192m 37% 10943Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2886Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12147m 40% 9814Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1970m 6% 5447Mi 4% gke-xlou-cdm-ds-0f601751-j488 11710m 39% 9155Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2211Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16464m 55% 10624Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13121m 43% 5651Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6655m 41% 5385Mi 9% 00:49:49 DEBUG --- stderr --- 00:49:49 DEBUG 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052246" 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052246" 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052246" 00:50:46 ERROR [http_cmd]: ERROR 00:50:46 ERROR --- status code --- 00:50:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:50:46 ERROR --- http response --- 00:50:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:50:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052246" 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "2297.9666666666667" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "instance": "10.142.0.107:9100" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "instance": "10.142.0.43:9100" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "0" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052246" 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052246" 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052246" 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "0.011" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "instance": "10.142.0.107:9100" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "instance": "10.142.0.43:9100" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "12588753920" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052246" 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "pod": "ds-idrepo-1" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "pod": "ds-cts-1" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "0" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "instance": "10.142.0.107:9100" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "instance": "10.142.0.43:9100" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "0.000500000000000019" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052246" 00:50:46 INFO [http_cmd]: http status code OK 00:50:46 DEBUG --- status code --- 00:50:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:46 DEBUG --- http response --- 00:50:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:46 DEBUG ----- output ----- 00:50:46 DEBUG { 00:50:46 DEBUG "status": "success", 00:50:46 DEBUG "data": { 00:50:46 DEBUG "resultType": "vector", 00:50:46 DEBUG "result": [ 00:50:46 DEBUG { 00:50:46 DEBUG "metric": { 00:50:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG [...] 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG 00:50:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:46 DEBUG }, 00:50:46 DEBUG "value": [ 00:50:46 DEBUG 1713052246, 00:50:46 DEBUG "2302" 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG ] 00:50:46 DEBUG } 00:50:46 DEBUG } 00:50:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052246" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052246, 00:50:47 DEBUG "2300.0666666666666" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "2298.5333333333333" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "ds-idrepo-1" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "ds-cts-1" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0.00021505555555555557" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG "data": { 00:50:47 DEBUG "data": { 00:50:47 DEBUG ----- output ----- 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG { 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG "result": [ 00:50:47 DEBUG "result": [ 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG { 00:50:47 DEBUG { 00:50:47 DEBUG "data": { 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:50:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:50:47 DEBUG "result": [ 00:50:47 DEBUG }, 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG }, 00:50:47 DEBUG { 00:50:47 DEBUG "value": [ 00:50:47 DEBUG { 00:50:47 DEBUG "value": [ 00:50:47 DEBUG "metric": { 00:50:47 DEBUG 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG 00:50:47 DEBUG "instance": "10.142.0.107:9100" 00:50:47 DEBUG 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 DEBUG "data": { 00:50:47 DEBUG 00:50:47 DEBUG }, 00:50:47 DEBUG 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG 00:50:47 DEBUG "value": [ 00:50:47 DEBUG [...] 00:50:47 DEBUG "result": [ 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG { 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "metric": { 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "ds-idrepo-1" 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:47 DEBUG }, 00:50:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:50:47 DEBUG 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG }, 00:50:47 DEBUG 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "instance": "10.142.0.43:9100" 00:50:47 DEBUG "413945" 00:50:47 DEBUG 00:50:47 DEBUG "0" 00:50:47 DEBUG }, 00:50:47 DEBUG ] 00:50:47 DEBUG [...] 00:50:47 DEBUG ] 00:50:47 DEBUG "value": [ 00:50:47 DEBUG } 00:50:47 DEBUG 00:50:47 DEBUG } 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG ] 00:50:47 DEBUG 00:50:47 DEBUG ] 00:50:47 DEBUG "5248.099999999999" 00:50:47 DEBUG } 00:50:47 DEBUG 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG "pod": "ds-idrepo-0" 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 DEBUG }, 00:50:47 DEBUG ] 00:50:47 DEBUG "value": [ 00:50:47 DEBUG } 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG } 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "ds-idrepo-1" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "ds-idrepo-0" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "ds-idrepo-1" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "ds-idrepo-0" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "ds-idrepo-1" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "ds-idrepo-0" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "19882.04742373128" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "6.6003394183554684" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "1783808000" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052247" 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "instance": "10.142.0.107:9100" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "instance": "10.142.0.43:9100" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "11951.999999999998" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "ds-idrepo-1" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "ds-idrepo-0" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "0" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:47 INFO [http_cmd]: http status code OK 00:50:47 DEBUG --- status code --- 00:50:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:50:47 DEBUG --- http response --- 00:50:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:50:47 DEBUG ----- output ----- 00:50:47 DEBUG { 00:50:47 DEBUG "status": "success", 00:50:47 DEBUG "data": { 00:50:47 DEBUG "resultType": "vector", 00:50:47 DEBUG "result": [ 00:50:47 DEBUG { 00:50:47 DEBUG "metric": { 00:50:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG [...] 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG 00:50:47 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 00:50:47 DEBUG }, 00:50:47 DEBUG "value": [ 00:50:47 DEBUG 1713052247, 00:50:47 DEBUG "6.574232597407977" 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG ] 00:50:47 DEBUG } 00:50:47 DEBUG } 00:50:48 INFO 00:50:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:50:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:50:48 INFO [loop_until]: OK (rc = 0) 00:50:48 DEBUG --- stdout --- 00:50:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12286m 8998Mi am-7d597cf7d6-8vr8f 11206m 8409Mi am-7d597cf7d6-9wz9h 15059m 9104Mi ds-cts-0 11353m 7533Mi ds-cts-1 11729m 7758Mi ds-cts-2 11588m 8014Mi ds-idrepo-0 13096m 4269Mi ds-idrepo-1 2002m 4291Mi ds-idrepo-2 1961m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 963Mi idm-b85f9d978-rl76f 6m 968Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6547m 1607Mi 00:50:48 DEBUG --- stderr --- 00:50:48 DEBUG 00:50:49 INFO 00:50:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:50:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:50:49 INFO [loop_until]: OK (rc = 0) 00:50:49 DEBUG --- stdout --- 00:50:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 4801Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11569m 38% 10123Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12807m 42% 11060Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17452m 58% 2989Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2062m 6% 6296Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11321m 37% 10943Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12324m 41% 9838Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1992m 6% 5449Mi 4% gke-xlou-cdm-ds-0f601751-j488 11620m 38% 9168Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 60m 0% 2216Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16440m 55% 10635Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13131m 43% 5647Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6772m 42% 5386Mi 9% 00:50:49 DEBUG --- stderr --- 00:50:49 DEBUG 00:51:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052306" 00:51:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052306" 00:51:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052306" 00:51:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052306" 00:51:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052306" 00:51:46 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052306" 00:51:46 ERROR [http_cmd]: ERROR 00:51:46 INFO [http_cmd]: http status code OK 00:51:46 ERROR --- status code --- 00:51:46 DEBUG --- status code --- 00:51:46 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:51:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:46 ERROR --- http response --- 00:51:46 DEBUG --- http response --- 00:51:46 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:51:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:46 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:51:46 DEBUG ----- output ----- 00:51:46 DEBUG { 00:51:46 DEBUG "status": "success", 00:51:46 DEBUG "data": { 00:51:46 DEBUG "resultType": "vector", 00:51:46 DEBUG "result": [ 00:51:46 DEBUG { 00:51:46 DEBUG "metric": { 00:51:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG [...] 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 1713052306, 00:51:46 DEBUG "2285.9666666666667" 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG } 00:51:46 INFO [http_cmd]: http status code OK 00:51:46 DEBUG --- status code --- 00:51:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:46 DEBUG --- http response --- 00:51:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:46 DEBUG ----- output ----- 00:51:46 DEBUG { 00:51:46 DEBUG "status": "success", 00:51:46 DEBUG "data": { 00:51:46 DEBUG "resultType": "vector", 00:51:46 DEBUG "result": [ 00:51:46 DEBUG { 00:51:46 DEBUG "metric": { 00:51:46 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG [...] 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 1713052306, 00:51:46 DEBUG "0.011" 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG } 00:51:46 INFO [http_cmd]: http status code OK 00:51:46 DEBUG --- status code --- 00:51:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:46 DEBUG --- http response --- 00:51:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:46 DEBUG ----- output ----- 00:51:46 DEBUG { 00:51:46 DEBUG "status": "success", 00:51:46 DEBUG "data": { 00:51:46 DEBUG "resultType": "vector", 00:51:46 DEBUG "result": [ 00:51:46 DEBUG { 00:51:46 DEBUG "metric": { 00:51:46 DEBUG "instance": "10.142.0.107:9100" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG [...] 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG "instance": "10.142.0.43:9100" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 1713052306, 00:51:46 DEBUG "0.00033333333333333365" 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG } 00:51:46 INFO [http_cmd]: http status code OK 00:51:46 DEBUG --- status code --- 00:51:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:46 DEBUG --- http response --- 00:51:46 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:51:46 INFO [http_cmd]: http status code OK 00:51:46 DEBUG --- status code --- 00:51:46 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:46 DEBUG --- http response --- 00:51:46 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:51:46 DEBUG ----- output ----- 00:51:46 DEBUG { 00:51:46 DEBUG "status": "success", 00:51:46 DEBUG "data": { 00:51:46 DEBUG "resultType": "vector", 00:51:46 DEBUG "result": [ 00:51:46 DEBUG { 00:51:46 DEBUG "metric": { 00:51:46 DEBUG "pod": "ds-idrepo-1" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG [...] 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG 00:51:46 DEBUG "pod": "ds-cts-1" 00:51:46 DEBUG }, 00:51:46 DEBUG "value": [ 00:51:46 DEBUG 1713052306, 00:51:46 DEBUG "0" 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG ] 00:51:46 DEBUG } 00:51:46 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "instance": "10.142.0.107:9100" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "instance": "10.142.0.43:9100" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "12589353984" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "instance": "10.142.0.107:9100" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "instance": "10.142.0.43:9100" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0.0004333333333332708" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "2291.4333333333334" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "2292.5333333333333" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "2292.2666666666664" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "ds-idrepo-1" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-cts-1" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0.0001" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:47 DEBUG { 00:51:47 DEBUG { 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG { 00:51:47 DEBUG "data": { 00:51:47 DEBUG "data": { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "result": [ 00:51:47 DEBUG "result": [ 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG { 00:51:47 DEBUG { 00:51:47 DEBUG "result": [ 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG { 00:51:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:51:47 DEBUG "metric": { 00:51:47 DEBUG }, 00:51:47 DEBUG }, 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG "instance": "10.142.0.107:9100" 00:51:47 DEBUG "value": [ 00:51:47 DEBUG "value": [ 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG }, 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:51:47 DEBUG [...] 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG { 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "status": "success", 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "data": { 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:51:47 DEBUG 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG }, 00:51:47 DEBUG }, 00:51:47 DEBUG "instance": "10.142.0.43:9100" 00:51:47 DEBUG "result": [ 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG "value": [ 00:51:47 DEBUG "value": [ 00:51:47 DEBUG }, 00:51:47 DEBUG { 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "414041.5" 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG "0" 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "pod": "ds-idrepo-1" 00:51:47 DEBUG ] 00:51:47 DEBUG { 00:51:47 DEBUG ] 00:51:47 DEBUG "5351.3" 00:51:47 DEBUG }, 00:51:47 DEBUG } 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG "value": [ 00:51:47 DEBUG ] 00:51:47 DEBUG "data": { 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG 00:51:47 DEBUG } 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 DEBUG "result": [ 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 DEBUG 00:51:47 DEBUG { 00:51:47 DEBUG } 00:51:47 DEBUG [...] 00:51:47 DEBUG "metric": { 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-1" 00:51:47 DEBUG 00:51:47 DEBUG }, 00:51:47 DEBUG 00:51:47 DEBUG "value": [ 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG 00:51:47 DEBUG }, 00:51:47 DEBUG 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG [...] 00:51:47 DEBUG "0" 00:51:47 DEBUG 00:51:47 DEBUG ] 00:51:47 DEBUG 00:51:47 DEBUG } 00:51:47 DEBUG 00:51:47 DEBUG ] 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG } 00:51:47 DEBUG }, 00:51:47 DEBUG } 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "ds-idrepo-1" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "ds-idrepo-1" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG { 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:51:47 DEBUG 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG 00:51:47 DEBUG { 00:51:47 DEBUG 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG "data": { 00:51:47 DEBUG }, 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "value": [ 00:51:47 DEBUG "result": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG { 00:51:47 DEBUG "201859.26840361248" 00:51:47 DEBUG "metric": { 00:51:47 DEBUG ] 00:51:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:51:47 DEBUG } 00:51:47 DEBUG }, 00:51:47 DEBUG ] 00:51:47 DEBUG "value": [ 00:51:47 DEBUG } 00:51:47 DEBUG 00:51:47 DEBUG } 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "4923.396790332012" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "6.541133718258146" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "1782382592" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052307" 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "instance": "10.142.0.107:9100" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "instance": "10.142.0.43:9100" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "12077.399999999998" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "ds-idrepo-1" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "0" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:47 INFO [http_cmd]: http status code OK 00:51:47 DEBUG --- status code --- 00:51:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:51:47 DEBUG --- http response --- 00:51:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:51:47 DEBUG ----- output ----- 00:51:47 DEBUG { 00:51:47 DEBUG "status": "success", 00:51:47 DEBUG "data": { 00:51:47 DEBUG "resultType": "vector", 00:51:47 DEBUG "result": [ 00:51:47 DEBUG { 00:51:47 DEBUG "metric": { 00:51:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG [...] 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG 00:51:47 DEBUG "pod": "ds-idrepo-0" 00:51:47 DEBUG }, 00:51:47 DEBUG "value": [ 00:51:47 DEBUG 1713052307, 00:51:47 DEBUG "13.177109641998566" 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG ] 00:51:47 DEBUG } 00:51:47 DEBUG } 00:51:48 INFO 00:51:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:51:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:51:48 INFO [loop_until]: OK (rc = 0) 00:51:48 DEBUG --- stdout --- 00:51:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12521m 8998Mi am-7d597cf7d6-8vr8f 11516m 8400Mi am-7d597cf7d6-9wz9h 15813m 9092Mi ds-cts-0 11403m 7520Mi ds-cts-1 11727m 7747Mi ds-cts-2 11213m 8021Mi ds-idrepo-0 13111m 4275Mi ds-idrepo-1 1968m 4291Mi ds-idrepo-2 1904m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 964Mi idm-b85f9d978-rl76f 6m 968Mi lodemon-64cbdcc7b4-vt4kk 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6511m 1609Mi 00:51:48 DEBUG --- stderr --- 00:51:48 DEBUG 00:51:49 INFO 00:51:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:51:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:51:49 INFO [loop_until]: OK (rc = 0) 00:51:49 DEBUG --- stdout --- 00:51:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4799Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11254m 37% 10131Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12767m 42% 11057Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17525m 58% 3001Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2086m 6% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11377m 38% 10973Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 59m 0% 2891Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12254m 41% 9834Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1983m 6% 5446Mi 4% gke-xlou-cdm-ds-0f601751-j488 11810m 39% 9180Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2209Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16256m 54% 10617Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13118m 43% 5659Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6691m 42% 5383Mi 9% 00:51:49 DEBUG --- stderr --- 00:51:49 DEBUG 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 ERROR [http_cmd]: ERROR 00:52:47 ERROR --- status code --- 00:52:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:52:47 ERROR --- http response --- 00:52:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:52:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0.011" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "instance": "10.142.0.107:9100" 00:52:47 DEBUG }, 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG "instance": "10.142.0.43:9100" 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-cts-1" 00:52:47 DEBUG { 00:52:47 DEBUG }, 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "value": [ 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG "data": { 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "12590017536" 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG "result": [ 00:52:47 DEBUG ] 00:52:47 DEBUG "2298.9" 00:52:47 DEBUG ] 00:52:47 DEBUG { 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG "metric": { 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG "instance": "10.142.0.107:9100" 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG }, 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 DEBUG "value": [ 00:52:47 DEBUG } 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "instance": "10.142.0.43:9100" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "instance": "10.142.0.107:9100" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "instance": "10.142.0.43:9100" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0.0004333333333332708" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "2307.8333333333335" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "2296.233333333333" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "2296.9666666666667" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "413914" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-cts-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0.0001" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG { 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG { 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "instance": "10.142.0.107:9100" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "instance": "10.142.0.43:9100" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "5526.133333333333" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:52:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:52:47 DEBUG }, 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG "pod": "ds-cts-2" 00:52:47 DEBUG }, 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG "41136681.87772261" 00:52:47 DEBUG ] 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "6.557629700655868" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "1784614912" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052367" 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:52:47 DEBUG "result": [ 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:52:47 DEBUG "data": { 00:52:47 DEBUG }, 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "value": [ 00:52:47 DEBUG "result": [ 00:52:47 DEBUG 00:52:47 DEBUG { 00:52:47 DEBUG 00:52:47 DEBUG "metric": { 00:52:47 DEBUG 00:52:47 DEBUG "instance": "10.142.0.107:9100" 00:52:47 DEBUG [...] 00:52:47 DEBUG }, 00:52:47 DEBUG 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-cts-2" 00:52:47 DEBUG 00:52:47 DEBUG }, 00:52:47 DEBUG [...] 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG 00:52:47 DEBUG "30462401.31598989" 00:52:47 DEBUG 00:52:47 DEBUG ] 00:52:47 DEBUG "instance": "10.142.0.43:9100" 00:52:47 DEBUG } 00:52:47 DEBUG }, 00:52:47 DEBUG ] 00:52:47 DEBUG "value": [ 00:52:47 DEBUG } 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG } 00:52:47 DEBUG "11751.733333333332" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "ds-idrepo-1" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:47 INFO [http_cmd]: http status code OK 00:52:47 DEBUG --- status code --- 00:52:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:52:47 DEBUG --- http response --- 00:52:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:52:47 DEBUG ----- output ----- 00:52:47 DEBUG { 00:52:47 DEBUG "status": "success", 00:52:47 DEBUG "data": { 00:52:47 DEBUG "resultType": "vector", 00:52:47 DEBUG "result": [ 00:52:47 DEBUG { 00:52:47 DEBUG "metric": { 00:52:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG [...] 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG 00:52:47 DEBUG "pod": "ds-idrepo-0" 00:52:47 DEBUG }, 00:52:47 DEBUG "value": [ 00:52:47 DEBUG 1713052367, 00:52:47 DEBUG "0" 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG ] 00:52:47 DEBUG } 00:52:47 DEBUG } 00:52:48 INFO 00:52:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:52:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:52:48 INFO [loop_until]: OK (rc = 0) 00:52:48 DEBUG --- stdout --- 00:52:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12144m 8998Mi am-7d597cf7d6-8vr8f 11436m 8392Mi am-7d597cf7d6-9wz9h 14704m 9092Mi ds-cts-0 11344m 7580Mi ds-cts-1 11987m 7798Mi ds-cts-2 11192m 8013Mi ds-idrepo-0 13388m 4274Mi ds-idrepo-1 2029m 4291Mi ds-idrepo-2 1894m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 19m 964Mi idm-b85f9d978-rl76f 9m 971Mi lodemon-64cbdcc7b4-vt4kk 1m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6366m 1611Mi 00:52:48 DEBUG --- stderr --- 00:52:48 DEBUG 00:52:49 INFO 00:52:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:52:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:52:49 INFO [loop_until]: OK (rc = 0) 00:52:49 DEBUG --- stdout --- 00:52:49 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 4800Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11488m 38% 10131Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12835m 42% 11055Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17498m 58% 2990Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2071m 6% 6293Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11509m 38% 11032Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 82m 0% 2899Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12322m 41% 9829Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1996m 6% 5443Mi 4% gke-xlou-cdm-ds-0f601751-j488 11820m 39% 9222Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2218Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16529m 55% 10617Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13146m 44% 5664Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6692m 42% 5386Mi 9% 00:52:49 DEBUG --- stderr --- 00:52:49 DEBUG 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 ERROR [http_cmd]: ERROR 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 ERROR --- status code --- 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG --- status code --- 00:53:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 ERROR --- http response --- 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:53:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG { 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "data": { 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG "result": [ 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG { 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG }, 00:53:47 DEBUG }, 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG "value": [ 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG [...] 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG "pod": "ds-cts-1" 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG }, 00:53:47 DEBUG }, 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG "value": [ 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0.011" 00:53:47 DEBUG "0" 00:53:47 DEBUG "2302.1666666666665" 00:53:47 DEBUG ] 00:53:47 DEBUG ] 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG ] 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "instance": "10.142.0.107:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "instance": "10.142.0.43:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "12590742528" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "instance": "10.142.0.107:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "instance": "10.142.0.43:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "instance": "10.142.0.107:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "instance": "10.142.0.43:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0.00043333333333350765" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "2299.5333333333333" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "2296.4" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "2298.8" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-idrepo-0" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "414084" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-cts-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0.00013333333333333334" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-idrepo-0" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-idrepo-0" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "instance": "10.142.0.107:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "instance": "10.142.0.43:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "6435.633333333333" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-idrepo-0" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG --- http response --- 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG { 00:53:47 DEBUG { 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "data": { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:53:47 DEBUG }, 00:53:47 DEBUG "result": [ 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG { 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG "metric": { 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG }, 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG "value": [ 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-cts-2" 00:53:47 DEBUG [...] 00:53:47 DEBUG "pod": "ds-cts-2" 00:53:47 DEBUG }, 00:53:47 DEBUG 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "355222.90536577936" 00:53:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:53:47 DEBUG "22710414.351559464" 00:53:47 DEBUG ] 00:53:47 DEBUG }, 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG "value": [ 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG "6.528161258411008" 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:53:47 DEBUG }, 00:53:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052427" 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "1783410688" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "instance": "10.142.0.107:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "instance": "10.142.0.43:9100" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "11895.699999999997" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-cts-2" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "4608204.074671541" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "ds-idrepo-1" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 INFO [http_cmd]: http status code OK 00:53:47 DEBUG "pod": "ds-idrepo-0" 00:53:47 DEBUG --- status code --- 00:53:47 DEBUG }, 00:53:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:53:47 DEBUG "value": [ 00:53:47 DEBUG --- http response --- 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "0" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:53:47 DEBUG ----- output ----- 00:53:47 DEBUG { 00:53:47 DEBUG "status": "success", 00:53:47 DEBUG "data": { 00:53:47 DEBUG "resultType": "vector", 00:53:47 DEBUG "result": [ 00:53:47 DEBUG { 00:53:47 DEBUG "metric": { 00:53:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG [...] 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG 00:53:47 DEBUG "pod": "ds-cts-2" 00:53:47 DEBUG }, 00:53:47 DEBUG "value": [ 00:53:47 DEBUG 1713052427, 00:53:47 DEBUG "6.135326849563931" 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG ] 00:53:47 DEBUG } 00:53:47 DEBUG } 00:53:48 INFO 00:53:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:53:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:53:48 INFO [loop_until]: OK (rc = 0) 00:53:48 DEBUG --- stdout --- 00:53:48 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12342m 8995Mi am-7d597cf7d6-8vr8f 10975m 8394Mi am-7d597cf7d6-9wz9h 15670m 9095Mi ds-cts-0 11172m 7584Mi ds-cts-1 11804m 7829Mi ds-cts-2 11553m 8029Mi ds-idrepo-0 12986m 4274Mi ds-idrepo-1 1935m 4291Mi ds-idrepo-2 1892m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 968Mi idm-b85f9d978-rl76f 5m 972Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6491m 1611Mi 00:53:48 DEBUG --- stderr --- 00:53:48 DEBUG 00:53:49 INFO 00:53:49 INFO [loop_until]: kubectl --namespace=xlou top node 00:53:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:53:50 INFO [loop_until]: OK (rc = 0) 00:53:50 DEBUG --- stdout --- 00:53:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 4807Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11544m 38% 10146Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12831m 42% 11047Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17363m 58% 2990Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2080m 6% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11338m 37% 11029Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2904Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11740m 39% 9825Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1931m 6% 5453Mi 4% gke-xlou-cdm-ds-0f601751-j488 11591m 38% 9238Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2212Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16435m 55% 10623Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13035m 43% 5669Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6671m 41% 5383Mi 9% 00:53:50 DEBUG --- stderr --- 00:53:50 DEBUG 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 ERROR [http_cmd]: ERROR 00:54:47 ERROR --- status code --- 00:54:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:54:47 ERROR --- http response --- 00:54:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:54:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG { 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG "0.011" 00:54:47 DEBUG { 00:54:47 DEBUG ] 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG } 00:54:47 DEBUG "data": { 00:54:47 DEBUG ] 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG } 00:54:47 DEBUG "result": [ 00:54:47 DEBUG } 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "2291.9333333333334" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG { 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG "data": { 00:54:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:54:47 DEBUG "result": [ 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "instance": "10.142.0.107:9100" 00:54:47 DEBUG "data": { 00:54:47 DEBUG }, 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "value": [ 00:54:47 DEBUG "result": [ 00:54:47 DEBUG 00:54:47 DEBUG { 00:54:47 DEBUG 00:54:47 DEBUG "metric": { 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG [...] 00:54:47 DEBUG }, 00:54:47 DEBUG 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "instance": "10.142.0.43:9100" 00:54:47 DEBUG 00:54:47 DEBUG }, 00:54:47 DEBUG [...] 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG 00:54:47 DEBUG "12591416320" 00:54:47 DEBUG 00:54:47 DEBUG ] 00:54:47 DEBUG "pod": "ds-cts-1" 00:54:47 DEBUG } 00:54:47 DEBUG }, 00:54:47 DEBUG ] 00:54:47 DEBUG "value": [ 00:54:47 DEBUG } 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG } 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "instance": "10.142.0.107:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "instance": "10.142.0.43:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0.0003333333333333327" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-0" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG "instance": "10.142.0.107:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG "instance": "10.142.0.43:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "2287.9" 00:54:47 DEBUG "0.00036666666666675953" 00:54:47 DEBUG ] 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "2288.0333333333333" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG { 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG "metric": { 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG { 00:54:47 DEBUG }, 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "value": [ 00:54:47 DEBUG "data": { 00:54:47 DEBUG 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG 00:54:47 DEBUG "result": [ 00:54:47 DEBUG 00:54:47 DEBUG { 00:54:47 DEBUG [...] 00:54:47 DEBUG "metric": { 00:54:47 DEBUG 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG 00:54:47 DEBUG }, 00:54:47 DEBUG 00:54:47 DEBUG "value": [ 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG 00:54:47 DEBUG }, 00:54:47 DEBUG 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG [...] 00:54:47 DEBUG "413535.5" 00:54:47 DEBUG 00:54:47 DEBUG ] 00:54:47 DEBUG 00:54:47 DEBUG } 00:54:47 DEBUG 00:54:47 DEBUG ] 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG } 00:54:47 DEBUG }, 00:54:47 DEBUG } 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "2287.5666666666666" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-cts-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0.00020591666666666667" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-0" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-2" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "5511.0743117367" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-0" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "instance": "10.142.0.107:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "instance": "10.142.0.43:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "5298.433333333333" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-0" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-2" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "148604847.0296005" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "6.560173290752935" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "ds-idrepo-1" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-0" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052487" 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "1786380288" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "instance": "10.142.0.107:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "instance": "10.142.0.43:9100" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "11745.833333333332" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 INFO [http_cmd]: http status code OK 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG --- status code --- 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG --- http response --- 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-2" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "1.8554153039398409" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:54:47 DEBUG ----- output ----- 00:54:47 DEBUG { 00:54:47 DEBUG "status": "success", 00:54:47 DEBUG "data": { 00:54:47 DEBUG "resultType": "vector", 00:54:47 DEBUG "result": [ 00:54:47 DEBUG { 00:54:47 DEBUG "metric": { 00:54:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG [...] 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG 00:54:47 DEBUG "pod": "ds-idrepo-2" 00:54:47 DEBUG }, 00:54:47 DEBUG "value": [ 00:54:47 DEBUG 1713052487, 00:54:47 DEBUG "0" 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG ] 00:54:47 DEBUG } 00:54:47 DEBUG } 00:54:48 INFO 00:54:48 INFO [loop_until]: kubectl --namespace=xlou top pods 00:54:48 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:54:49 INFO [loop_until]: OK (rc = 0) 00:54:49 DEBUG --- stdout --- 00:54:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12628m 8997Mi am-7d597cf7d6-8vr8f 11375m 8391Mi am-7d597cf7d6-9wz9h 14980m 9093Mi ds-cts-0 11540m 7580Mi ds-cts-1 11919m 7806Mi ds-cts-2 11926m 8027Mi ds-idrepo-0 13077m 4275Mi ds-idrepo-1 2018m 4290Mi ds-idrepo-2 1901m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 968Mi idm-b85f9d978-rl76f 4m 971Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6604m 1613Mi 00:54:49 DEBUG --- stderr --- 00:54:49 DEBUG 00:54:50 INFO 00:54:50 INFO [loop_until]: kubectl --namespace=xlou top node 00:54:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:54:50 INFO [loop_until]: OK (rc = 0) 00:54:50 DEBUG --- stdout --- 00:54:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 4806Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11661m 39% 10148Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12853m 43% 11054Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17476m 58% 3000Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2082m 6% 6294Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11410m 38% 11034Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2906Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12409m 41% 9824Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1879m 6% 5450Mi 4% gke-xlou-cdm-ds-0f601751-j488 11741m 39% 9235Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2199Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16387m 54% 10623Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13132m 43% 5674Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6697m 42% 5383Mi 9% 00:54:50 DEBUG --- stderr --- 00:54:50 DEBUG 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0.012" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 ERROR [http_cmd]: ERROR 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 ERROR --- status code --- 00:55:47 DEBUG --- status code --- 00:55:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 ERROR --- http response --- 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:55:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "2297.0333333333333" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "instance": "10.142.0.107:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "instance": "10.142.0.43:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "instance": "10.142.0.107:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "instance": "10.142.0.43:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "12591995904" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-cts-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "instance": "10.142.0.107:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "instance": "10.142.0.43:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0.0004333333333332708" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-idrepo-0" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "2297.1" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "2297.233333333333" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-cts-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0.00007795833333333334" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "413452.5" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-cts-2" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "39318045.820295244" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "2297.5333333333333" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-idrepo-0" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG --- status code --- 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG { 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG "data": { 00:55:47 DEBUG { 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG "result": [ 00:55:47 DEBUG "data": { 00:55:47 DEBUG { 00:55:47 DEBUG { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "result": [ 00:55:47 DEBUG "data": { 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG }, 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "result": [ 00:55:47 DEBUG "value": [ 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG { 00:55:47 DEBUG 00:55:47 DEBUG }, 00:55:47 DEBUG "metric": { 00:55:47 DEBUG 00:55:47 DEBUG "value": [ 00:55:47 DEBUG "instance": "10.142.0.107:9100" 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG }, 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-idrepo-0" 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG }, 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "value": [ 00:55:47 DEBUG "pod": "ds-idrepo-0" 00:55:47 DEBUG 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG }, 00:55:47 DEBUG 00:55:47 DEBUG "0" 00:55:47 DEBUG "value": [ 00:55:47 DEBUG "instance": "10.142.0.43:9100" 00:55:47 DEBUG ] 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG }, 00:55:47 DEBUG } 00:55:47 DEBUG "0" 00:55:47 DEBUG "value": [ 00:55:47 DEBUG ] 00:55:47 DEBUG ] 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 DEBUG "5300.833333333333" 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG }, 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG "value": [ 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG "pod": "ds-cts-2" 00:55:47 DEBUG { 00:55:47 DEBUG }, 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "value": [ 00:55:47 DEBUG "data": { 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "5889.64824856572" 00:55:47 DEBUG "result": [ 00:55:47 DEBUG ] 00:55:47 DEBUG { 00:55:47 DEBUG } 00:55:47 DEBUG "metric": { 00:55:47 DEBUG ] 00:55:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:55:47 DEBUG } 00:55:47 DEBUG }, 00:55:47 DEBUG } 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "6.6556272220012085" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "1788092416" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "ds-idrepo-1" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-idrepo-0" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052547" 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "instance": "10.142.0.107:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "instance": "10.142.0.43:9100" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "11814.8" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 INFO [http_cmd]: http status code OK 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG --- status code --- 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG --- http response --- 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-cts-2" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "0" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:55:47 DEBUG ----- output ----- 00:55:47 DEBUG { 00:55:47 DEBUG "status": "success", 00:55:47 DEBUG "data": { 00:55:47 DEBUG "resultType": "vector", 00:55:47 DEBUG "result": [ 00:55:47 DEBUG { 00:55:47 DEBUG "metric": { 00:55:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG [...] 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG 00:55:47 DEBUG "pod": "ds-cts-2" 00:55:47 DEBUG }, 00:55:47 DEBUG "value": [ 00:55:47 DEBUG 1713052547, 00:55:47 DEBUG "8.334974665155018" 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG ] 00:55:47 DEBUG } 00:55:47 DEBUG } 00:55:49 INFO 00:55:49 INFO [loop_until]: kubectl --namespace=xlou top pods 00:55:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:55:49 INFO [loop_until]: OK (rc = 0) 00:55:49 DEBUG --- stdout --- 00:55:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12249m 8996Mi am-7d597cf7d6-8vr8f 11411m 8393Mi am-7d597cf7d6-9wz9h 15147m 9095Mi ds-cts-0 11130m 7605Mi ds-cts-1 11577m 7858Mi ds-cts-2 11432m 8054Mi ds-idrepo-0 13216m 4274Mi ds-idrepo-1 1983m 4291Mi ds-idrepo-2 1871m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 968Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6668m 1614Mi 00:55:49 DEBUG --- stderr --- 00:55:49 DEBUG 00:55:50 INFO 00:55:50 INFO [loop_until]: kubectl --namespace=xlou top node 00:55:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:55:50 INFO [loop_until]: OK (rc = 0) 00:55:50 DEBUG --- stdout --- 00:55:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4812Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11191m 37% 10140Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12802m 42% 11053Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17459m 58% 2993Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 1973m 6% 6302Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11301m 37% 11043Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2902Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12281m 41% 9828Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1992m 6% 5444Mi 4% gke-xlou-cdm-ds-0f601751-j488 11702m 39% 9286Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2200Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16459m 55% 10623Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13261m 44% 5668Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6800m 42% 5382Mi 9% 00:55:50 DEBUG --- stderr --- 00:55:50 DEBUG 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0.012" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 ERROR [http_cmd]: ERROR 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 ERROR --- status code --- 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 ERROR --- http response --- 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:56:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG ----- output ----- 00:56:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG { 00:56:47 DEBUG { 00:56:47 DEBUG { 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG "data": { 00:56:47 DEBUG "data": { 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG "result": [ 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG { 00:56:47 DEBUG { 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG "data": { 00:56:47 DEBUG "instance": "10.142.0.107:9100" 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG }, 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG }, 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG "result": [ 00:56:47 DEBUG "value": [ 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG { 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "metric": { 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "instance": "10.142.0.107:9100" 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG }, 00:56:47 DEBUG [...] 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-cts-1" 00:56:47 DEBUG 00:56:47 DEBUG "instance": "10.142.0.43:9100" 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG }, 00:56:47 DEBUG [...] 00:56:47 DEBUG }, 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG "value": [ 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG 00:56:47 DEBUG "12592524288" 00:56:47 DEBUG "2286.9666666666667" 00:56:47 DEBUG ] 00:56:47 DEBUG "instance": "10.142.0.43:9100" 00:56:47 DEBUG ] 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG }, 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG "value": [ 00:56:47 DEBUG ] 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 DEBUG "0.00033333333333333175" 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "413094" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG { 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "2286.3" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG } 00:56:47 DEBUG ----- output ----- 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG { 00:56:47 DEBUG --- status code --- 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG "data": { 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:56:47 DEBUG "result": [ 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:56:47 DEBUG { 00:56:47 DEBUG { 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG { 00:56:47 DEBUG "instance": "10.142.0.107:9100" 00:56:47 DEBUG "data": { 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG }, 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG "data": { 00:56:47 DEBUG "value": [ 00:56:47 DEBUG "result": [ 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG 00:56:47 DEBUG { 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG "result": [ 00:56:47 DEBUG 00:56:47 DEBUG "metric": { 00:56:47 DEBUG { 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG "metric": { 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG [...] 00:56:47 DEBUG }, 00:56:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:56:47 DEBUG { 00:56:47 DEBUG 00:56:47 DEBUG "value": [ 00:56:47 DEBUG }, 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "value": [ 00:56:47 DEBUG "data": { 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "instance": "10.142.0.43:9100" 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "result": [ 00:56:47 DEBUG }, 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG { 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG "metric": { 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG "0.00040000000000001514" 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG }, 00:56:47 DEBUG ] 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG 00:56:47 DEBUG "value": [ 00:56:47 DEBUG } 00:56:47 DEBUG }, 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG 00:56:47 DEBUG ] 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG "value": [ 00:56:47 DEBUG }, 00:56:47 DEBUG 00:56:47 DEBUG } 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG } 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG "0" 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG [...] 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG ] 00:56:47 DEBUG "3609.584224204914" 00:56:47 DEBUG 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG 00:56:47 DEBUG { 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 DEBUG }, 00:56:47 DEBUG "data": { 00:56:47 DEBUG } 00:56:47 DEBUG "value": [ 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "result": [ 00:56:47 DEBUG "2306.2" 00:56:47 DEBUG { 00:56:47 DEBUG ] 00:56:47 DEBUG "metric": { 00:56:47 DEBUG } 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG ] 00:56:47 DEBUG }, 00:56:47 DEBUG } 00:56:47 DEBUG "value": [ 00:56:47 DEBUG } 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-cts-1" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0.00016881944444444442" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "2285.5333333333333" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "instance": "10.142.0.107:9100" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "instance": "10.142.0.43:9100" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "5438.733333333333" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "7962.318141628486" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "6.528485012249276" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "1790959616" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "8.442092416268142" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "ds-idrepo-1" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052607" 00:56:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052607" 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "instance": "10.142.0.107:9100" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "instance": "10.142.0.43:9100" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "12203.83333333333" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:47 INFO [http_cmd]: http status code OK 00:56:47 DEBUG --- status code --- 00:56:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:56:47 DEBUG --- http response --- 00:56:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:56:47 DEBUG ----- output ----- 00:56:47 DEBUG { 00:56:47 DEBUG "status": "success", 00:56:47 DEBUG "data": { 00:56:47 DEBUG "resultType": "vector", 00:56:47 DEBUG "result": [ 00:56:47 DEBUG { 00:56:47 DEBUG "metric": { 00:56:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG [...] 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG 00:56:47 DEBUG "pod": "ds-idrepo-0" 00:56:47 DEBUG }, 00:56:47 DEBUG "value": [ 00:56:47 DEBUG 1713052607, 00:56:47 DEBUG "0" 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG ] 00:56:47 DEBUG } 00:56:47 DEBUG } 00:56:49 INFO 00:56:49 INFO [loop_until]: kubectl --namespace=xlou top pods 00:56:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:56:49 INFO [loop_until]: OK (rc = 0) 00:56:49 DEBUG --- stdout --- 00:56:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12484m 8994Mi am-7d597cf7d6-8vr8f 11154m 8393Mi am-7d597cf7d6-9wz9h 15198m 9106Mi ds-cts-0 10801m 7600Mi ds-cts-1 11923m 7853Mi ds-cts-2 11412m 8037Mi ds-idrepo-0 13027m 4276Mi ds-idrepo-1 1985m 4290Mi ds-idrepo-2 1794m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 7m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 7m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6493m 1617Mi 00:56:49 DEBUG --- stderr --- 00:56:49 DEBUG 00:56:50 INFO 00:56:50 INFO [loop_until]: kubectl --namespace=xlou top node 00:56:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:56:50 INFO [loop_until]: OK (rc = 0) 00:56:50 DEBUG --- stdout --- 00:56:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 4809Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11524m 38% 10148Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12853m 43% 11055Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 16856m 56% 2993Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2036m 6% 6293Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11401m 38% 11025Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2903Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12313m 41% 9817Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1986m 6% 5457Mi 4% gke-xlou-cdm-ds-0f601751-j488 11823m 39% 9286Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2212Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16422m 55% 10625Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 12713m 42% 5659Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6665m 41% 5385Mi 9% 00:56:50 DEBUG --- stderr --- 00:56:50 DEBUG 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 ERROR [http_cmd]: ERROR 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 ERROR --- status code --- 00:57:47 DEBUG --- status code --- 00:57:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 ERROR --- http response --- 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:57:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0.012" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "result": [ 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG { 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG "metric": { 00:57:47 DEBUG { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "instance": "10.142.0.107:9100" 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG }, 00:57:47 DEBUG "data": { 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG "value": [ 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "result": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG { 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG "metric": { 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG "instance": "10.142.0.107:9100" 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG }, 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-cts-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "instance": "10.142.0.43:9100" 00:57:47 DEBUG 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG }, 00:57:47 DEBUG 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG [...] 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "2305.4" 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG "12593155072" 00:57:47 DEBUG 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG "instance": "10.142.0.43:9100" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG }, 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG "value": [ 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "412871.5" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "2307.2" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "2285.4" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-idrepo-0" 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG }, 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG "2307.9333333333334" 00:57:47 DEBUG ] 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-cts-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0.00006666666666666667" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "instance": "10.142.0.107:9100" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG 00:57:47 DEBUG { 00:57:47 DEBUG 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG [...] 00:57:47 DEBUG "data": { 00:57:47 DEBUG 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG 00:57:47 DEBUG "result": [ 00:57:47 DEBUG 00:57:47 DEBUG { 00:57:47 DEBUG "instance": "10.142.0.43:9100" 00:57:47 DEBUG "metric": { 00:57:47 DEBUG }, 00:57:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:57:47 DEBUG "value": [ 00:57:47 DEBUG }, 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG "0.000500000000000019" 00:57:47 DEBUG 00:57:47 DEBUG ] 00:57:47 DEBUG 00:57:47 DEBUG } 00:57:47 DEBUG 00:57:47 DEBUG ] 00:57:47 DEBUG [...] 00:57:47 DEBUG } 00:57:47 DEBUG 00:57:47 DEBUG } 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "idm-b85f9d978-rl76f" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "550.8893446757003" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-idrepo-0" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "instance": "10.142.0.107:9100" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "instance": "10.142.0.43:9100" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "5327" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-idrepo-0" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "1830125568" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-idrepo-0" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "6.536427536844579" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "ds-idrepo-1" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "ds-idrepo-0" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052667" 00:57:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052667" 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "instance": "10.142.0.107:9100" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "instance": "10.142.0.43:9100" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "12134.266666666665" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:47 INFO [http_cmd]: http status code OK 00:57:47 DEBUG --- status code --- 00:57:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:57:47 DEBUG --- http response --- 00:57:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:57:47 DEBUG ----- output ----- 00:57:47 DEBUG { 00:57:47 DEBUG "status": "success", 00:57:47 DEBUG "data": { 00:57:47 DEBUG "resultType": "vector", 00:57:47 DEBUG "result": [ 00:57:47 DEBUG { 00:57:47 DEBUG "metric": { 00:57:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG [...] 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG 00:57:47 DEBUG "pod": "idm-b85f9d978-rl76f" 00:57:47 DEBUG }, 00:57:47 DEBUG "value": [ 00:57:47 DEBUG 1713052667, 00:57:47 DEBUG "0" 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG ] 00:57:47 DEBUG } 00:57:47 DEBUG } 00:57:49 INFO 00:57:49 INFO [loop_until]: kubectl --namespace=xlou top pods 00:57:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:57:49 INFO [loop_until]: OK (rc = 0) 00:57:49 DEBUG --- stdout --- 00:57:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12379m 8997Mi am-7d597cf7d6-8vr8f 11165m 8392Mi am-7d597cf7d6-9wz9h 15352m 9095Mi ds-cts-0 11061m 7590Mi ds-cts-1 11662m 7871Mi ds-cts-2 11434m 8037Mi ds-idrepo-0 13089m 4276Mi ds-idrepo-1 1876m 4290Mi ds-idrepo-2 1844m 4105Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 7m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6623m 1653Mi 00:57:49 DEBUG --- stderr --- 00:57:49 DEBUG 00:57:50 INFO 00:57:50 INFO [loop_until]: kubectl --namespace=xlou top node 00:57:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:57:50 INFO [loop_until]: OK (rc = 0) 00:57:50 DEBUG --- stdout --- 00:57:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4813Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11732m 39% 10158Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12923m 43% 11052Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17429m 58% 3006Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2096m 7% 6300Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11191m 37% 11046Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 62m 0% 2901Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12187m 40% 9824Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1936m 6% 5443Mi 4% gke-xlou-cdm-ds-0f601751-j488 11962m 40% 9302Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2199Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16536m 55% 10630Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13196m 44% 5657Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6695m 42% 5420Mi 9% 00:57:50 DEBUG --- stderr --- 00:57:50 DEBUG 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG --- http response --- 00:58:47 ERROR [http_cmd]: ERROR 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:58:47 ERROR --- status code --- 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG ----- output ----- 00:58:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:58:47 DEBUG { 00:58:47 DEBUG { 00:58:47 ERROR --- http response --- 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG "pod": "ds-cts-1" 00:58:47 DEBUG }, 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0.012" 00:58:47 DEBUG "0" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG ] 00:58:47 DEBUG ] 00:58:47 DEBUG --- status code --- 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG ] 00:58:47 DEBUG ] 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG } 00:58:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:58:47 DEBUG } 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "instance": "10.142.0.107:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "instance": "10.142.0.43:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "12593875968" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "instance": "10.142.0.107:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "instance": "10.142.0.43:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0.000333333333333341" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG }, 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG }, 00:58:47 DEBUG }, 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG "value": [ 00:58:47 DEBUG "value": [ 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "2297.6666666666665" 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG "413281.5" 00:58:47 DEBUG ] 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG { 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG "data": { 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG } 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "2290.7" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "5430.232908493777" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "2286.9666666666667" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "2287.133333333333" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-cts-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0.00017795833333333335" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "instance": "10.142.0.107:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "instance": "10.142.0.43:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0.000500000000000019" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "instance": "10.142.0.107:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "instance": "10.142.0.43:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "6776.133333333333" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "1831165952" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "17401.42818403688" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "6.538077622649515" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "ds-idrepo-1" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "13.15470719214195" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052727" 00:58:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052727" 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 INFO [http_cmd]: http status code OK 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG --- status code --- 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG --- http response --- 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "instance": "10.142.0.107:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "instance": "10.142.0.43:9100" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "12250.066666666664" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 00:58:47 DEBUG ----- output ----- 00:58:47 DEBUG { 00:58:47 DEBUG "status": "success", 00:58:47 DEBUG "data": { 00:58:47 DEBUG "resultType": "vector", 00:58:47 DEBUG "result": [ 00:58:47 DEBUG { 00:58:47 DEBUG "metric": { 00:58:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG [...] 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG 00:58:47 DEBUG "pod": "ds-idrepo-0" 00:58:47 DEBUG }, 00:58:47 DEBUG "value": [ 00:58:47 DEBUG 1713052727, 00:58:47 DEBUG "0" 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG ] 00:58:47 DEBUG } 00:58:47 DEBUG } 00:58:49 INFO 00:58:49 INFO [loop_until]: kubectl --namespace=xlou top pods 00:58:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:58:49 INFO [loop_until]: OK (rc = 0) 00:58:49 DEBUG --- stdout --- 00:58:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 13164m 8995Mi am-7d597cf7d6-8vr8f 11844m 8391Mi am-7d597cf7d6-9wz9h 15399m 9091Mi ds-cts-0 10926m 7597Mi ds-cts-1 11593m 7875Mi ds-cts-2 11487m 8065Mi ds-idrepo-0 13409m 4276Mi ds-idrepo-1 2033m 4291Mi ds-idrepo-2 1906m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 10m 969Mi idm-b85f9d978-rl76f 5m 972Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6632m 1654Mi 00:58:49 DEBUG --- stderr --- 00:58:49 DEBUG 00:58:50 INFO 00:58:50 INFO [loop_until]: kubectl --namespace=xlou top node 00:58:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:58:50 INFO [loop_until]: OK (rc = 0) 00:58:50 DEBUG --- stdout --- 00:58:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4801Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11317m 37% 10159Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12535m 41% 11051Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17483m 58% 2993Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2045m 6% 6292Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11295m 37% 11053Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 69m 0% 2903Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12383m 41% 9822Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1994m 6% 5449Mi 4% gke-xlou-cdm-ds-0f601751-j488 11848m 39% 9299Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2187Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16318m 54% 10630Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13227m 44% 5664Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6695m 42% 5420Mi 9% 00:58:50 DEBUG --- stderr --- 00:58:50 DEBUG 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "instance": "10.142.0.107:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "instance": "10.142.0.43:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 DEBUG --- status code --- 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG --- http response --- 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 DEBUG "metric": { 00:59:47 DEBUG { 00:59:47 ERROR [http_cmd]: ERROR 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG "metric": { 00:59:47 ERROR --- status code --- 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:59:47 DEBUG }, 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG }, 00:59:47 ERROR --- http response --- 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 00:59:47 DEBUG 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 00:59:47 DEBUG 00:59:47 DEBUG ----- output ----- 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG { 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG { 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG "data": { 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG "data": { 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:59:47 DEBUG "resultType": "vector", 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "result": [ 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG "result": [ 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG { 00:59:47 DEBUG { 00:59:47 DEBUG { 00:59:47 DEBUG { 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG }, 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG }, 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "metric": { 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG "value": [ 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG }, 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG "value": [ 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG "data": { 00:59:47 DEBUG "data": { 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG "value": [ 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG }, 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG }, 00:59:47 DEBUG "2286.233333333333" 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG "8103160.114322007" 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG "value": [ 00:59:47 DEBUG "result": [ 00:59:47 DEBUG "result": [ 00:59:47 DEBUG "value": [ 00:59:47 DEBUG ] 00:59:47 DEBUG { 00:59:47 DEBUG "413193" 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG ] 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG 00:59:47 DEBUG { 00:59:47 DEBUG { 00:59:47 DEBUG 00:59:47 DEBUG } 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG ] 00:59:47 DEBUG { 00:59:47 DEBUG } 00:59:47 DEBUG { 00:59:47 DEBUG 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG 00:59:47 DEBUG ] 00:59:47 DEBUG "data": { 00:59:47 DEBUG } 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG ] 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG 00:59:47 DEBUG "instance": "10.142.0.107:9100" 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG 00:59:47 DEBUG } 00:59:47 DEBUG "resultType": "vector", 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG ] 00:59:47 DEBUG "data": { 00:59:47 DEBUG } 00:59:47 DEBUG "data": { 00:59:47 DEBUG [...] 00:59:47 DEBUG }, 00:59:47 DEBUG }, 00:59:47 DEBUG [...] 00:59:47 DEBUG } 00:59:47 DEBUG "result": [ 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG } 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG { 00:59:47 DEBUG } 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG "result": [ 00:59:47 DEBUG "result": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "metric": { 00:59:47 DEBUG } 00:59:47 DEBUG { 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 00:59:47 DEBUG { 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG "metric": { 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG }, 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG { 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG }, 00:59:47 DEBUG [...] 00:59:47 DEBUG "pod": "ds-cts-1" 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 DEBUG }, 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG }, 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 DEBUG "data": { 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG 00:59:47 DEBUG "0.012" 00:59:47 DEBUG 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "pod": "ds-idrepo-0" 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "result": [ 00:59:47 DEBUG 00:59:47 DEBUG ] 00:59:47 DEBUG "instance": "10.142.0.43:9100" 00:59:47 DEBUG "0" 00:59:47 DEBUG }, 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG { 00:59:47 DEBUG 00:59:47 DEBUG } 00:59:47 DEBUG }, 00:59:47 DEBUG ] 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG "metric": { 00:59:47 DEBUG [...] 00:59:47 DEBUG ] 00:59:47 DEBUG "value": [ 00:59:47 DEBUG } 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG 00:59:47 DEBUG } 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG ] 00:59:47 DEBUG "0" 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG }, 00:59:47 DEBUG 00:59:47 DEBUG } 00:59:47 DEBUG "12594570240" 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 DEBUG }, 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 DEBUG 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG "value": [ 00:59:47 DEBUG }, 00:59:47 DEBUG 00:59:47 DEBUG }, 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG "value": [ 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 DEBUG "2293.633333333333" 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG [...] 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG "2294.4333333333334" 00:59:47 DEBUG 00:59:47 DEBUG "2298.8333333333335" 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG 00:59:47 DEBUG ] 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG "pod": "ds-cts-1" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 DEBUG }, 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 DEBUG "value": [ 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0.00019999999999999998" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "instance": "10.142.0.107:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "instance": "10.142.0.43:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0.0004666666666665264" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "ds-idrepo-0" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "ds-idrepo-0" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "instance": "10.142.0.107:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "instance": "10.142.0.43:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "5216.933333333333" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "ds-idrepo-0" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "1834881024" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "299643536.86884725" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "6.5261596955188645" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "instance": "10.142.0.107:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "instance": "10.142.0.43:9100" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "11984.999999999998" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "ds-idrepo-1" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "ds-idrepo-0" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "12.368761257542046" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052787" 00:59:47 INFO [http_cmd]: http status code OK 00:59:47 DEBUG --- status code --- 00:59:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 00:59:47 DEBUG --- http response --- 00:59:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 00:59:47 DEBUG ----- output ----- 00:59:47 DEBUG { 00:59:47 DEBUG "status": "success", 00:59:47 DEBUG "data": { 00:59:47 DEBUG "resultType": "vector", 00:59:47 DEBUG "result": [ 00:59:47 DEBUG { 00:59:47 DEBUG "metric": { 00:59:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG [...] 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG 00:59:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 00:59:47 DEBUG }, 00:59:47 DEBUG "value": [ 00:59:47 DEBUG 1713052787, 00:59:47 DEBUG "0" 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG ] 00:59:47 DEBUG } 00:59:47 DEBUG } 00:59:49 INFO 00:59:49 INFO [loop_until]: kubectl --namespace=xlou top pods 00:59:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:59:49 INFO [loop_until]: OK (rc = 0) 00:59:49 DEBUG --- stdout --- 00:59:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12383m 8995Mi am-7d597cf7d6-8vr8f 11379m 8391Mi am-7d597cf7d6-9wz9h 15312m 9098Mi ds-cts-0 11176m 7596Mi ds-cts-1 11797m 7878Mi ds-cts-2 11237m 8052Mi ds-idrepo-0 13457m 4276Mi ds-idrepo-1 1909m 4291Mi ds-idrepo-2 1894m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 9m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6624m 1656Mi 00:59:49 DEBUG --- stderr --- 00:59:49 DEBUG 00:59:50 INFO 00:59:50 INFO [loop_until]: kubectl --namespace=xlou top node 00:59:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 00:59:50 INFO [loop_until]: OK (rc = 0) 00:59:50 DEBUG --- stdout --- 00:59:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4807Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11296m 37% 10164Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12792m 42% 11060Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17430m 58% 2987Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2078m 6% 6297Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11401m 38% 11046Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2890Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12168m 40% 9818Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 2003m 6% 5455Mi 4% gke-xlou-cdm-ds-0f601751-j488 11803m 39% 9303Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 58m 0% 2198Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16512m 55% 10620Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13167m 44% 5672Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6703m 42% 5421Mi 9% 00:59:50 DEBUG --- stderr --- 00:59:50 DEBUG 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG { 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-cts-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0.00015591666666666667" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "413712.5" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "instance": "10.142.0.107:9100" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "instance": "10.142.0.43:9100" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0.0003333333333333327" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052847" 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "2297.7" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG { 01:00:47 DEBUG { 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG "data": { 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG { 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG "result": [ 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG { 01:00:47 DEBUG 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 DEBUG "metric": { 01:00:47 ERROR [http_cmd]: ERROR 01:00:47 DEBUG 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG 01:00:47 DEBUG "instance": "10.142.0.107:9100" 01:00:47 ERROR --- status code --- 01:00:47 DEBUG 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG { 01:00:47 DEBUG [...] 01:00:47 DEBUG }, 01:00:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:00:47 DEBUG [...] 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG 01:00:47 DEBUG "value": [ 01:00:47 ERROR --- http response --- 01:00:47 DEBUG 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG "data": { 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG "result": [ 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG }, 01:00:47 DEBUG [...] 01:00:47 DEBUG }, 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG "value": [ 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "instance": "10.142.0.107:9100" 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "data": { 01:00:47 DEBUG }, 01:00:47 DEBUG "2297.6666666666665" 01:00:47 DEBUG 01:00:47 DEBUG "0" 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "value": [ 01:00:47 DEBUG ] 01:00:47 DEBUG "instance": "10.142.0.43:9100" 01:00:47 DEBUG ] 01:00:47 DEBUG "result": [ 01:00:47 DEBUG 01:00:47 DEBUG } 01:00:47 DEBUG }, 01:00:47 DEBUG } 01:00:47 DEBUG { 01:00:47 DEBUG 01:00:47 DEBUG ] 01:00:47 DEBUG "value": [ 01:00:47 DEBUG ] 01:00:47 DEBUG "metric": { 01:00:47 DEBUG 01:00:47 DEBUG } 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG } 01:00:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:00:47 DEBUG [...] 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 DEBUG "12595682304" 01:00:47 DEBUG } 01:00:47 DEBUG }, 01:00:47 DEBUG 01:00:47 DEBUG ] 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG } 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG ] 01:00:47 DEBUG 01:00:47 DEBUG "instance": "10.142.0.43:9100" 01:00:47 DEBUG } 01:00:47 DEBUG 01:00:47 DEBUG }, 01:00:47 DEBUG } 01:00:47 DEBUG [...] 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG 01:00:47 DEBUG "0.00033333333333350387" 01:00:47 DEBUG 01:00:47 DEBUG ] 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG } 01:00:47 DEBUG }, 01:00:47 DEBUG ] 01:00:47 DEBUG "value": [ 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG } 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG "2880.4066250339397" 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG ] 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "2309" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0.012" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-cts-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "2297.633333333333" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "instance": "10.142.0.107:9100" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "instance": "10.142.0.43:9100" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "5254.233333333333" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "1836920832" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG "data": { 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG "result": [ 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG 01:00:47 DEBUG { 01:00:47 DEBUG 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG "data": { 01:00:47 DEBUG }, 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "value": [ 01:00:47 DEBUG "result": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG { 01:00:47 DEBUG "1706.907629649742" 01:00:47 DEBUG "metric": { 01:00:47 DEBUG ] 01:00:47 DEBUG "instance": "10.142.0.107:9100" 01:00:47 DEBUG } 01:00:47 DEBUG }, 01:00:47 DEBUG ] 01:00:47 DEBUG "value": [ 01:00:47 DEBUG } 01:00:47 DEBUG 01:00:47 DEBUG } 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "instance": "10.142.0.43:9100" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "11636.699999999999" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "6.615328142041418" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "ds-idrepo-1" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "6.29730174619885" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052847" 01:00:47 INFO [http_cmd]: http status code OK 01:00:47 DEBUG --- status code --- 01:00:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:00:47 DEBUG --- http response --- 01:00:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:00:47 DEBUG ----- output ----- 01:00:47 DEBUG { 01:00:47 DEBUG "status": "success", 01:00:47 DEBUG "data": { 01:00:47 DEBUG "resultType": "vector", 01:00:47 DEBUG "result": [ 01:00:47 DEBUG { 01:00:47 DEBUG "metric": { 01:00:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG [...] 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG 01:00:47 DEBUG "pod": "ds-idrepo-0" 01:00:47 DEBUG }, 01:00:47 DEBUG "value": [ 01:00:47 DEBUG 1713052847, 01:00:47 DEBUG "0" 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG ] 01:00:47 DEBUG } 01:00:47 DEBUG } 01:00:49 INFO 01:00:49 INFO [loop_until]: kubectl --namespace=xlou top pods 01:00:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:00:49 INFO [loop_until]: OK (rc = 0) 01:00:49 DEBUG --- stdout --- 01:00:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12399m 9010Mi am-7d597cf7d6-8vr8f 11365m 8396Mi am-7d597cf7d6-9wz9h 15435m 9094Mi ds-cts-0 11522m 7697Mi ds-cts-1 11980m 7905Mi ds-cts-2 11228m 8050Mi ds-idrepo-0 13157m 4277Mi ds-idrepo-1 2003m 4293Mi ds-idrepo-2 1858m 4106Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6471m 1657Mi 01:00:49 DEBUG --- stderr --- 01:00:49 DEBUG 01:00:50 INFO 01:00:50 INFO [loop_until]: kubectl --namespace=xlou top node 01:00:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:00:50 INFO [loop_until]: OK (rc = 0) 01:00:50 DEBUG --- stdout --- 01:00:50 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4803Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11671m 39% 10167Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12911m 43% 11069Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 16941m 56% 2987Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2068m 6% 6303Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11379m 38% 11163Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 66m 0% 2890Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12339m 41% 9827Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 2007m 6% 5441Mi 4% gke-xlou-cdm-ds-0f601751-j488 11872m 39% 9321Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 58m 0% 2185Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16483m 55% 10623Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13200m 44% 5667Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6738m 42% 5421Mi 9% 01:00:50 DEBUG --- stderr --- 01:00:50 DEBUG 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-cts-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG "0.0001" 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG ] 01:01:47 DEBUG { 01:01:47 DEBUG } 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG ] 01:01:47 DEBUG "data": { 01:01:47 DEBUG } 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG } 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "2296" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "instance": "10.142.0.107:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "instance": "10.142.0.43:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG "instance": "10.142.0.107:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG "instance": "10.142.0.43:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "413376" 01:01:47 DEBUG "0.0003999999999997783" 01:01:47 DEBUG ] 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG { 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG "data": { 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG }, 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "value": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG }, 01:01:47 DEBUG 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG [...] 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG }, 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG "0.012" 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG }, 01:01:47 DEBUG ] 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG } 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG ] 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "2296.4333333333334" 01:01:47 DEBUG } 01:01:47 DEBUG "2286.7666666666664" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "instance": "10.142.0.107:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "instance": "10.142.0.43:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "12596347904" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 ERROR [http_cmd]: ERROR 01:01:47 ERROR --- status code --- 01:01:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:01:47 ERROR --- http response --- 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:01:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-cts-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "3385.129479888326" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "2296.133333333333" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "instance": "10.142.0.107:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "instance": "10.142.0.43:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "5312.599999999999" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "3166.73402957295" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "instance": "10.142.0.107:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "instance": "10.142.0.43:9100" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "11762.999999999998" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "1836077056" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "ds-idrepo-1" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "6.549585814363828" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052907" 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "6.760445107483696" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:47 INFO [http_cmd]: http status code OK 01:01:47 DEBUG --- status code --- 01:01:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:01:47 DEBUG --- http response --- 01:01:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:01:47 DEBUG ----- output ----- 01:01:47 DEBUG { 01:01:47 DEBUG "status": "success", 01:01:47 DEBUG "data": { 01:01:47 DEBUG "resultType": "vector", 01:01:47 DEBUG "result": [ 01:01:47 DEBUG { 01:01:47 DEBUG "metric": { 01:01:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG [...] 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG 01:01:47 DEBUG "pod": "ds-idrepo-0" 01:01:47 DEBUG }, 01:01:47 DEBUG "value": [ 01:01:47 DEBUG 1713052907, 01:01:47 DEBUG "0" 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG ] 01:01:47 DEBUG } 01:01:47 DEBUG } 01:01:49 INFO 01:01:49 INFO [loop_until]: kubectl --namespace=xlou top pods 01:01:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:01:49 INFO [loop_until]: OK (rc = 0) 01:01:49 DEBUG --- stdout --- 01:01:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12594m 9026Mi am-7d597cf7d6-8vr8f 10988m 8412Mi am-7d597cf7d6-9wz9h 15557m 9111Mi ds-cts-0 11409m 7701Mi ds-cts-1 11025m 7908Mi ds-cts-2 11743m 8048Mi ds-idrepo-0 13095m 4276Mi ds-idrepo-1 2031m 4293Mi ds-idrepo-2 1846m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6533m 1659Mi 01:01:49 DEBUG --- stderr --- 01:01:49 DEBUG 01:01:50 INFO 01:01:50 INFO [loop_until]: kubectl --namespace=xlou top node 01:01:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:01:51 INFO [loop_until]: OK (rc = 0) 01:01:51 DEBUG --- stdout --- 01:01:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4811Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11658m 39% 10168Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12928m 43% 11087Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17486m 58% 2997Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2086m 6% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11221m 37% 11141Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 69m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12151m 40% 9842Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1981m 6% 5450Mi 4% gke-xlou-cdm-ds-0f601751-j488 11652m 39% 9327Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2190Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16455m 55% 10621Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13196m 44% 5668Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6679m 42% 5416Mi 9% 01:01:51 DEBUG --- stderr --- 01:01:51 DEBUG 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG --- status code --- 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG ----- output ----- 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 ERROR [http_cmd]: ERROR 01:02:47 DEBUG ----- output ----- 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG { 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- status code --- 01:02:47 ERROR --- status code --- 01:02:47 DEBUG { 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG "data": { 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG --- http response --- 01:02:47 ERROR --- http response --- 01:02:47 DEBUG "data": { 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG "resultType": "vector", 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG { 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG "result": [ 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG ----- output ----- 01:02:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG "result": [ 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG { 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG { 01:02:47 DEBUG { 01:02:47 DEBUG { 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG { 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 DEBUG { 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG "data": { 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG "metric": { 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:02:47 DEBUG "data": { 01:02:47 DEBUG "data": { 01:02:47 DEBUG { 01:02:47 DEBUG "data": { 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG "result": [ 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG "data": { 01:02:47 DEBUG }, 01:02:47 DEBUG "result": [ 01:02:47 DEBUG "data": { 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG }, 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG { 01:02:47 DEBUG "result": [ 01:02:47 DEBUG "result": [ 01:02:47 DEBUG "data": { 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "value": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG { 01:02:47 DEBUG "value": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "instance": "10.142.0.107:9100" 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG { 01:02:47 DEBUG { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG 01:02:47 DEBUG "metric": { 01:02:47 DEBUG }, 01:02:47 DEBUG "result": [ 01:02:47 DEBUG "data": { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "result": [ 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "data": { 01:02:47 DEBUG 01:02:47 DEBUG "data": { 01:02:47 DEBUG }, 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG "data": { 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG { 01:02:47 DEBUG "instance": "10.142.0.107:9100" 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "value": [ 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "value": [ 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "result": [ 01:02:47 DEBUG }, 01:02:47 DEBUG }, 01:02:47 DEBUG "metric": { 01:02:47 DEBUG }, 01:02:47 DEBUG "result": [ 01:02:47 DEBUG [...] 01:02:47 DEBUG "result": [ 01:02:47 DEBUG 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG "result": [ 01:02:47 DEBUG 01:02:47 DEBUG "metric": { 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG { 01:02:47 DEBUG "value": [ 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG { 01:02:47 DEBUG 01:02:47 DEBUG { 01:02:47 DEBUG 01:02:47 DEBUG { 01:02:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:02:47 DEBUG 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG "value": [ 01:02:47 DEBUG "pod": "ds-cts-1" 01:02:47 DEBUG "metric": { 01:02:47 DEBUG 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG "metric": { 01:02:47 DEBUG 01:02:47 DEBUG "metric": { 01:02:47 DEBUG 01:02:47 DEBUG "metric": { 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG [...] 01:02:47 DEBUG "instance": "10.142.0.107:9100" 01:02:47 DEBUG [...] 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG "value": [ 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG "value": [ 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG "value": [ 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG "0.0002" 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "value": [ 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG ] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "instance": "10.142.0.43:9100" 01:02:47 DEBUG 01:02:47 DEBUG } 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "0.012" 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG [...] 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "instance": "10.142.0.43:9100" 01:02:47 DEBUG [...] 01:02:47 DEBUG ] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG "pod": "ds-idrepo-0" 01:02:47 DEBUG [...] 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG 01:02:47 DEBUG "value": [ 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG } 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "0" 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG ] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG ] 01:02:47 DEBUG "0.00033333333333333365" 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG "value": [ 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG "2268.9666666666667" 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "12596902912" 01:02:47 DEBUG } 01:02:47 DEBUG 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG }, 01:02:47 DEBUG ] 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG ] 01:02:47 DEBUG 01:02:47 DEBUG } 01:02:47 DEBUG 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG "pod": "ds-cts-1" 01:02:47 DEBUG "0" 01:02:47 DEBUG "value": [ 01:02:47 DEBUG } 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG } 01:02:47 DEBUG "pod": "ds-idrepo-0" 01:02:47 DEBUG "instance": "10.142.0.43:9100" 01:02:47 DEBUG }, 01:02:47 DEBUG } 01:02:47 DEBUG "2267.9666666666667" 01:02:47 DEBUG "0" 01:02:47 DEBUG } 01:02:47 DEBUG }, 01:02:47 DEBUG ] 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG ] 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG ] 01:02:47 DEBUG }, 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "2274.6" 01:02:47 DEBUG } 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG } 01:02:47 DEBUG "value": [ 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG "0" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "412799.5" 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG "0" 01:02:47 DEBUG "0.000500000000000019" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG { 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG "data": { 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG "result": [ 01:02:47 DEBUG } 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "2297.9" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "ds-idrepo-0" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "0" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "instance": "10.142.0.107:9100" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "instance": "10.142.0.43:9100" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "5249.866666666667" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "ds-idrepo-0" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "0" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713052967" 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "ds-cts-2" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "0" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "instance": "10.142.0.107:9100" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "instance": "10.142.0.43:9100" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "11718.033333333331" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "1838206976" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "pod": "ds-idrepo-1" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "pod": "ds-idrepo-0" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "0" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713052967" 01:02:47 INFO [http_cmd]: http status code OK 01:02:47 DEBUG --- status code --- 01:02:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:47 DEBUG --- http response --- 01:02:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:47 DEBUG ----- output ----- 01:02:47 DEBUG { 01:02:47 DEBUG "status": "success", 01:02:47 DEBUG "data": { 01:02:47 DEBUG "resultType": "vector", 01:02:47 DEBUG "result": [ 01:02:47 DEBUG { 01:02:47 DEBUG "metric": { 01:02:47 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG [...] 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG 01:02:47 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:02:47 DEBUG }, 01:02:47 DEBUG "value": [ 01:02:47 DEBUG 1713052967, 01:02:47 DEBUG "6.503688489741772" 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG ] 01:02:47 DEBUG } 01:02:47 DEBUG } 01:02:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052968" 01:02:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713052968" 01:02:48 INFO [http_cmd]: http status code OK 01:02:48 DEBUG --- status code --- 01:02:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:48 DEBUG --- http response --- 01:02:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:02:48 DEBUG ----- output ----- 01:02:48 DEBUG { 01:02:48 DEBUG "status": "success", 01:02:48 DEBUG "data": { 01:02:48 DEBUG "resultType": "vector", 01:02:48 DEBUG "result": [ 01:02:48 DEBUG { 01:02:48 DEBUG "metric": { 01:02:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:02:48 DEBUG }, 01:02:48 DEBUG "value": [ 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG [...] 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG "pod": "ds-cts-2" 01:02:48 DEBUG }, 01:02:48 DEBUG "value": [ 01:02:48 DEBUG 1713052968, 01:02:48 DEBUG "0" 01:02:48 DEBUG ] 01:02:48 DEBUG } 01:02:48 DEBUG ] 01:02:48 DEBUG } 01:02:48 DEBUG } 01:02:48 INFO [http_cmd]: http status code OK 01:02:48 DEBUG --- status code --- 01:02:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:02:48 DEBUG --- http response --- 01:02:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:02:48 DEBUG ----- output ----- 01:02:48 DEBUG { 01:02:48 DEBUG "status": "success", 01:02:48 DEBUG "data": { 01:02:48 DEBUG "resultType": "vector", 01:02:48 DEBUG "result": [ 01:02:48 DEBUG { 01:02:48 DEBUG "metric": { 01:02:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:02:48 DEBUG }, 01:02:48 DEBUG "value": [ 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG [...] 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG 01:02:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:02:48 DEBUG }, 01:02:48 DEBUG "value": [ 01:02:48 DEBUG 1713052968, 01:02:48 DEBUG "0" 01:02:48 DEBUG ] 01:02:48 DEBUG } 01:02:48 DEBUG ] 01:02:48 DEBUG } 01:02:48 DEBUG } 01:02:49 INFO 01:02:49 INFO [loop_until]: kubectl --namespace=xlou top pods 01:02:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:02:49 INFO [loop_until]: OK (rc = 0) 01:02:49 DEBUG --- stdout --- 01:02:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12344m 8991Mi am-7d597cf7d6-8vr8f 11391m 8405Mi am-7d597cf7d6-9wz9h 15100m 9096Mi ds-cts-0 11188m 7727Mi ds-cts-1 11597m 7986Mi ds-cts-2 11437m 8063Mi ds-idrepo-0 13198m 4277Mi ds-idrepo-1 2025m 4293Mi ds-idrepo-2 1897m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6486m 1661Mi 01:02:49 DEBUG --- stderr --- 01:02:49 DEBUG 01:02:51 INFO 01:02:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:02:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:02:51 INFO [loop_until]: OK (rc = 0) 01:02:51 DEBUG --- stdout --- 01:02:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4816Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11229m 37% 10179Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12848m 43% 11053Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17433m 58% 2997Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2109m 7% 6298Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11221m 37% 11150Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2908Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12324m 41% 9843Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1945m 6% 5461Mi 4% gke-xlou-cdm-ds-0f601751-j488 11717m 39% 9423Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2205Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16373m 54% 10624Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13193m 44% 5681Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6676m 42% 5419Mi 9% 01:02:51 DEBUG --- stderr --- 01:02:51 DEBUG 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG --- status code --- 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG --- status code --- 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- http response --- 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- http response --- 01:03:47 ERROR [http_cmd]: ERROR 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 ERROR --- status code --- 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:03:47 DEBUG ----- output ----- 01:03:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:03:47 DEBUG { 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 ERROR --- http response --- 01:03:47 DEBUG "status": "success", 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG { 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG { 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:03:47 DEBUG { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG { 01:03:47 DEBUG { 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG "data": { 01:03:47 DEBUG "resultType": "vector", 01:03:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG "data": { 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG "data": { 01:03:47 DEBUG "data": { 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "data": { 01:03:47 DEBUG { 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "result": [ 01:03:47 DEBUG { 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "metric": { 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG "result": [ 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:03:47 DEBUG "result": [ 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG "result": [ 01:03:47 DEBUG { 01:03:47 DEBUG { 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "data": { 01:03:47 DEBUG { 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "metric": { 01:03:47 DEBUG { 01:03:47 DEBUG { 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "metric": { 01:03:47 DEBUG }, 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "data": { 01:03:47 DEBUG "pod": "ds-idrepo-1" 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "pod": "ds-idrepo-1" 01:03:47 DEBUG "value": [ 01:03:47 DEBUG { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "instance": "10.142.0.107:9100" 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG }, 01:03:47 DEBUG "instance": "10.142.0.107:9100" 01:03:47 DEBUG "pod": "ds-idrepo-1" 01:03:47 DEBUG { 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "data": { 01:03:47 DEBUG }, 01:03:47 DEBUG "data": { 01:03:47 DEBUG }, 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "value": [ 01:03:47 DEBUG }, 01:03:47 DEBUG }, 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG "data": { 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "value": [ 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "value": [ 01:03:47 DEBUG { 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG "value": [ 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG { 01:03:47 DEBUG "result": [ 01:03:47 DEBUG 01:03:47 DEBUG "result": [ 01:03:47 DEBUG 01:03:47 DEBUG "metric": { 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG "result": [ 01:03:47 DEBUG "metric": { 01:03:47 DEBUG { 01:03:47 DEBUG 01:03:47 DEBUG { 01:03:47 DEBUG 01:03:47 DEBUG "instance": "10.142.0.107:9100" 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG { 01:03:47 DEBUG "pod": "ds-idrepo-1" 01:03:47 DEBUG "metric": { 01:03:47 DEBUG 01:03:47 DEBUG "metric": { 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "metric": { 01:03:47 DEBUG }, 01:03:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:03:47 DEBUG [...] 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG [...] 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG "value": [ 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "pod": "ds-cts-1" 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG "pod": "ds-cts-1" 01:03:47 DEBUG "pod": "ds-idrepo-0" 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "instance": "10.142.0.43:9100" 01:03:47 DEBUG 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG [...] 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 DEBUG }, 01:03:47 DEBUG "instance": "10.142.0.43:9100" 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG "411971" 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG }, 01:03:47 DEBUG 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG ] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG "value": [ 01:03:47 DEBUG [...] 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG "0" 01:03:47 DEBUG } 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "0" 01:03:47 DEBUG "0.0005532361111111111" 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG "0" 01:03:47 DEBUG }, 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "12597623808" 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG "0.0005333333333335114" 01:03:47 DEBUG }, 01:03:47 DEBUG ] 01:03:47 DEBUG "value": [ 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG 01:03:47 DEBUG "pod": "ds-idrepo-0" 01:03:47 DEBUG 01:03:47 DEBUG ] 01:03:47 DEBUG 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG "value": [ 01:03:47 DEBUG } 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG 01:03:47 DEBUG }, 01:03:47 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:03:47 DEBUG } 01:03:47 DEBUG "instance": "10.142.0.43:9100" 01:03:47 DEBUG }, 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG ] 01:03:47 DEBUG "2302.2" 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG "value": [ 01:03:47 DEBUG }, 01:03:47 DEBUG ] 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG "2303.2" 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG }, 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG } 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG "value": [ 01:03:47 DEBUG "0" 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG } 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "2295.0333333333333" 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG ] 01:03:47 DEBUG "0" 01:03:47 DEBUG "0" 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG "2298.4666666666667" 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "result": [ 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "0.012" 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "result": [ 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "pod": "ds-idrepo-1" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG [...] 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "pod": "ds-idrepo-0" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "0" 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:47 DEBUG ] 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG } 01:03:47 DEBUG { 01:03:47 DEBUG } 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "result": [ 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "instance": "10.142.0.107:9100" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "instance": "10.142.0.43:9100" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "7012.566666666667" 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 INFO [http_cmd]: http status code OK 01:03:47 DEBUG --- status code --- 01:03:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:47 DEBUG --- http response --- 01:03:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:47 DEBUG ----- output ----- 01:03:47 DEBUG { 01:03:47 DEBUG "status": "success", 01:03:47 DEBUG "data": { 01:03:47 DEBUG "resultType": "vector", 01:03:47 DEBUG "result": [ 01:03:47 DEBUG { 01:03:47 DEBUG "metric": { 01:03:47 DEBUG "pod": "ds-idrepo-1" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG [...] 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG 01:03:47 DEBUG "pod": "ds-idrepo-0" 01:03:47 DEBUG }, 01:03:47 DEBUG "value": [ 01:03:47 DEBUG 1713053027, 01:03:47 DEBUG "0" 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG ] 01:03:47 DEBUG } 01:03:47 DEBUG } 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053027" 01:03:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053027" 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053028" 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053027, 01:03:48 DEBUG "30241.972067003528" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "instance": "10.142.0.107:9100" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "instance": "10.142.0.43:9100" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053027, 01:03:48 DEBUG "11865.499999999998" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053028, 01:03:48 DEBUG "1840115712" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053028" 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "pod": "ds-idrepo-1" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "pod": "ds-idrepo-0" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053028, 01:03:48 DEBUG "0" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053028" 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053028, 01:03:48 DEBUG "6.546030732109181" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053028" 01:03:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053028" 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053028, 01:03:48 DEBUG "13.011038635702402" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:48 INFO [http_cmd]: http status code OK 01:03:48 DEBUG --- status code --- 01:03:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:03:48 DEBUG --- http response --- 01:03:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:03:48 DEBUG ----- output ----- 01:03:48 DEBUG { 01:03:48 DEBUG "status": "success", 01:03:48 DEBUG "data": { 01:03:48 DEBUG "resultType": "vector", 01:03:48 DEBUG "result": [ 01:03:48 DEBUG { 01:03:48 DEBUG "metric": { 01:03:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG [...] 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG 01:03:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:03:48 DEBUG }, 01:03:48 DEBUG "value": [ 01:03:48 DEBUG 1713053028, 01:03:48 DEBUG "0" 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG ] 01:03:48 DEBUG } 01:03:48 DEBUG } 01:03:49 INFO 01:03:49 INFO [loop_until]: kubectl --namespace=xlou top pods 01:03:49 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:03:49 INFO [loop_until]: OK (rc = 0) 01:03:49 DEBUG --- stdout --- 01:03:49 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12798m 8992Mi am-7d597cf7d6-8vr8f 11514m 8392Mi am-7d597cf7d6-9wz9h 15307m 9097Mi ds-cts-0 10774m 7724Mi ds-cts-1 11679m 7980Mi ds-cts-2 12005m 8057Mi ds-idrepo-0 12893m 4277Mi ds-idrepo-1 1957m 4293Mi ds-idrepo-2 1902m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 969Mi idm-b85f9d978-rl76f 4m 972Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6644m 1661Mi 01:03:49 DEBUG --- stderr --- 01:03:49 DEBUG 01:03:51 INFO 01:03:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:03:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:03:51 INFO [loop_until]: OK (rc = 0) 01:03:51 DEBUG --- stdout --- 01:03:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4817Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11644m 39% 10178Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12966m 43% 11054Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17385m 58% 2995Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2106m 7% 6296Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11278m 37% 11165Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2899Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12163m 40% 9834Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1995m 6% 5448Mi 4% gke-xlou-cdm-ds-0f601751-j488 11837m 39% 9430Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 63m 0% 2192Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16004m 53% 10625Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13172m 44% 5678Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6681m 42% 5420Mi 9% 01:03:51 DEBUG --- stderr --- 01:03:51 DEBUG 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053087" 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 ERROR [http_cmd]: ERROR 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 DEBUG --- status code --- 01:04:47 ERROR --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 DEBUG --- http response --- 01:04:47 ERROR --- http response --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 DEBUG --- status code --- 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- http response --- 01:04:47 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG { 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG ----- output ----- 01:04:47 INFO [http_cmd]: http status code OK 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG { 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG --- status code --- 01:04:47 DEBUG { 01:04:47 DEBUG "data": { 01:04:47 DEBUG { 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG { 01:04:47 DEBUG "data": { 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG { 01:04:47 DEBUG "data": { 01:04:47 DEBUG "data": { 01:04:47 DEBUG "data": { 01:04:47 DEBUG --- http response --- 01:04:47 DEBUG "data": { 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "data": { 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG { 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:47 DEBUG { 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "data": { 01:04:47 DEBUG "data": { 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "data": { 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG "data": { 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "result": [ 01:04:47 DEBUG ----- output ----- 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "data": { 01:04:47 DEBUG "data": { 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG { 01:04:47 DEBUG "data": { 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:04:47 DEBUG { 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "status": "success", 01:04:47 DEBUG }, 01:04:47 DEBUG "metric": { 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG "result": [ 01:04:47 DEBUG "result": [ 01:04:47 DEBUG { 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG "result": [ 01:04:47 DEBUG { 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG "pod": "ds-idrepo-1" 01:04:47 DEBUG "instance": "10.142.0.107:9100" 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG "data": { 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "pod": "ds-idrepo-1" 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG { 01:04:47 DEBUG { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG }, 01:04:47 DEBUG { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG "resultType": "vector", 01:04:47 DEBUG 01:04:47 DEBUG }, 01:04:47 DEBUG "pod": "ds-idrepo-1" 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "metric": { 01:04:47 DEBUG "pod": "ds-idrepo-1" 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "result": [ 01:04:47 DEBUG 01:04:47 DEBUG "value": [ 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:04:47 DEBUG }, 01:04:47 DEBUG 01:04:47 DEBUG "instance": "10.142.0.107:9100" 01:04:47 DEBUG }, 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG { 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 01:04:47 DEBUG }, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "metric": { 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG "instance": "10.142.0.107:9100" 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG }, 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "pod": "ds-cts-1" 01:04:47 DEBUG "pod": "ds-idrepo-1" 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG [...] 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG 01:04:47 DEBUG "pod": "ds-cts-1" 01:04:47 DEBUG "pod": "ds-idrepo-0" 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG }, 01:04:47 DEBUG "instance": "10.142.0.43:9100" 01:04:47 DEBUG }, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG [...] 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG 01:04:47 DEBUG }, 01:04:47 DEBUG }, 01:04:47 DEBUG 01:04:47 DEBUG "instance": "10.142.0.43:9100" 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "0" 01:04:47 DEBUG 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 01:04:47 DEBUG }, 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "pod": "ds-idrepo-0" 01:04:47 DEBUG }, 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "12598266880" 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG 01:04:47 DEBUG 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:04:47 DEBUG "value": [ 01:04:47 DEBUG }, 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG }, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "2277.233333333333" 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG "2277.366666666667" 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "instance": "10.142.0.43:9100" 01:04:47 DEBUG "0.0002333333333333333" 01:04:47 DEBUG }, 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "0.012" 01:04:47 DEBUG "411888" 01:04:47 DEBUG "value": [ 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "4788.6665291245445" 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG "0" 01:04:47 DEBUG }, 01:04:47 DEBUG ] 01:04:47 DEBUG "value": [ 01:04:47 DEBUG "0" 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG "0.00036666666666675953" 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG "value": [ 01:04:47 DEBUG } 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG ] 01:04:47 DEBUG "2277.0666666666666" 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG "0" 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG 1713053087, 01:04:47 DEBUG ] 01:04:47 DEBUG "2297.4666666666667" 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG "0" 01:04:47 DEBUG ] 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG ] 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 DEBUG } 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053087" 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "pod": "ds-idrepo-1" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "pod": "ds-idrepo-0" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053087, 01:04:48 DEBUG "0" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "instance": "10.142.0.107:9100" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "instance": "10.142.0.43:9100" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053087, 01:04:48 DEBUG "5406.966666666666" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "pod": "ds-idrepo-1" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "pod": "ds-idrepo-0" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053087, 01:04:48 DEBUG "0" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053088" 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "pod": "ds-idrepo-1" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "pod": "ds-idrepo-0" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "0" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053088" 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053088" 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "instance": "10.142.0.107:9100" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "instance": "10.142.0.43:9100" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "11827.566666666664" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053088" 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "pod": "ds-idrepo-1" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "154862799.5323568" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "1842790400" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053088" 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "6.567843713947846" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053088" 01:04:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053088" 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "pod": "ds-idrepo-1" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "2.001762017742953" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:48 INFO [http_cmd]: http status code OK 01:04:48 DEBUG --- status code --- 01:04:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:04:48 DEBUG --- http response --- 01:04:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (116 lines): 01:04:48 DEBUG ----- output ----- 01:04:48 DEBUG { 01:04:48 DEBUG "status": "success", 01:04:48 DEBUG "data": { 01:04:48 DEBUG "resultType": "vector", 01:04:48 DEBUG "result": [ 01:04:48 DEBUG { 01:04:48 DEBUG "metric": { 01:04:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG [...] 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG 01:04:48 DEBUG "pod": "ds-idrepo-1" 01:04:48 DEBUG }, 01:04:48 DEBUG "value": [ 01:04:48 DEBUG 1713053088, 01:04:48 DEBUG "0" 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG ] 01:04:48 DEBUG } 01:04:48 DEBUG } 01:04:50 INFO 01:04:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:04:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:04:50 INFO [loop_until]: OK (rc = 0) 01:04:50 DEBUG --- stdout --- 01:04:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12310m 8999Mi am-7d597cf7d6-8vr8f 11472m 8394Mi am-7d597cf7d6-9wz9h 15004m 9094Mi ds-cts-0 11335m 7713Mi ds-cts-1 11679m 7987Mi ds-cts-2 11627m 8063Mi ds-idrepo-0 13434m 4276Mi ds-idrepo-1 1953m 4293Mi ds-idrepo-2 1888m 4112Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 7m 969Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6457m 1663Mi 01:04:50 DEBUG --- stderr --- 01:04:50 DEBUG 01:04:51 INFO 01:04:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:04:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:04:51 INFO [loop_until]: OK (rc = 0) 01:04:51 DEBUG --- stdout --- 01:04:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4810Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11709m 39% 10194Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12856m 43% 11050Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 16835m 56% 2987Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2090m 7% 6298Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11413m 38% 11159Mi 9% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2895Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12134m 40% 9833Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1971m 6% 5455Mi 4% gke-xlou-cdm-ds-0f601751-j488 11751m 39% 9425Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 68m 0% 2200Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16421m 55% 10628Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13225m 44% 5678Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6693m 42% 5422Mi 9% 01:04:51 DEBUG --- stderr --- 01:04:51 DEBUG 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: http status code OK 01:05:47 DEBUG --- status code --- 01:05:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:47 DEBUG --- http response --- 01:05:47 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053147" 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053147" 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: http status code OK 01:05:47 DEBUG --- status code --- 01:05:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:47 DEBUG --- http response --- 01:05:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:47 DEBUG ----- output ----- 01:05:47 DEBUG { 01:05:47 DEBUG "status": "success", 01:05:47 DEBUG "data": { 01:05:47 DEBUG "resultType": "vector", 01:05:47 DEBUG "result": [ 01:05:47 DEBUG { 01:05:47 DEBUG "metric": { 01:05:47 DEBUG "instance": "10.142.0.107:9100" 01:05:47 DEBUG }, 01:05:47 DEBUG "value": [ 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG [...] 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG "instance": "10.142.0.43:9100" 01:05:47 DEBUG }, 01:05:47 DEBUG "value": [ 01:05:47 DEBUG 1713053147, 01:05:47 DEBUG "0.00033333333333333365" 01:05:47 DEBUG ] 01:05:47 DEBUG } 01:05:47 DEBUG ] 01:05:47 DEBUG } 01:05:47 DEBUG } 01:05:47 INFO [http_cmd]: http status code OK 01:05:47 DEBUG --- status code --- 01:05:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:47 DEBUG --- http response --- 01:05:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:47 DEBUG ----- output ----- 01:05:47 DEBUG { 01:05:47 DEBUG "status": "success", 01:05:47 DEBUG "data": { 01:05:47 DEBUG "resultType": "vector", 01:05:47 DEBUG "result": [ 01:05:47 DEBUG { 01:05:47 DEBUG "metric": { 01:05:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:47 DEBUG }, 01:05:47 DEBUG "value": [ 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG [...] 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:47 DEBUG }, 01:05:47 DEBUG "value": [ 01:05:47 DEBUG 1713053147, 01:05:47 DEBUG "2290.233333333333" 01:05:47 DEBUG ] 01:05:47 DEBUG } 01:05:47 DEBUG ] 01:05:47 DEBUG } 01:05:47 DEBUG } 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: http status code OK 01:05:47 DEBUG --- status code --- 01:05:47 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:47 DEBUG --- http response --- 01:05:47 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:47 DEBUG ----- output ----- 01:05:47 DEBUG { 01:05:47 DEBUG "status": "success", 01:05:47 DEBUG "data": { 01:05:47 DEBUG "resultType": "vector", 01:05:47 DEBUG "result": [ 01:05:47 DEBUG { 01:05:47 DEBUG "metric": { 01:05:47 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:47 DEBUG }, 01:05:47 DEBUG "value": [ 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG [...] 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG 01:05:47 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:47 DEBUG }, 01:05:47 DEBUG "value": [ 01:05:47 DEBUG 1713053147, 01:05:47 DEBUG "2293.4666666666667" 01:05:47 DEBUG ] 01:05:47 DEBUG } 01:05:47 DEBUG ] 01:05:47 DEBUG } 01:05:47 DEBUG } 01:05:47 ERROR [http_cmd]: ERROR 01:05:47 ERROR --- status code --- 01:05:47 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:05:47 ERROR --- http response --- 01:05:47 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053147" 01:05:47 INFO [http_cmd]: http status code OK 01:05:47 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:05:47 DEBUG --- status code --- 01:05:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053147, 01:05:48 DEBUG "2293.1" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053147, 01:05:48 DEBUG "2271.733333333333" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053147, 01:05:48 DEBUG "0.012" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-idrepo-0" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053147, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 DEBUG --- status code --- 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053148" 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG { 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG { 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG "data": { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG { 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG { 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "instance": "10.142.0.107:9100" 01:05:48 DEBUG "data": { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:05:48 DEBUG "data": { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:05:48 DEBUG }, 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG "result": [ 01:05:48 DEBUG }, 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG "value": [ 01:05:48 DEBUG "result": [ 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG { 01:05:48 DEBUG "value": [ 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG 01:05:48 DEBUG { 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "metric": { 01:05:48 DEBUG 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG 01:05:48 DEBUG "metric": { 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "instance": "10.142.0.107:9100" 01:05:48 DEBUG 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG }, 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG }, 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG "value": [ 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG "value": [ 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG { 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "instance": "10.142.0.43:9100" 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:05:48 DEBUG "data": { 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-cts-1" 01:05:48 DEBUG }, 01:05:48 DEBUG [...] 01:05:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:05:48 DEBUG }, 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG [...] 01:05:48 DEBUG "pod": "ds-cts-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG "result": [ 01:05:48 DEBUG 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG { 01:05:48 DEBUG 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "12598770688" 01:05:48 DEBUG 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "413261.5" 01:05:48 DEBUG "metric": { 01:05:48 DEBUG 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG "instance": "10.142.0.43:9100" 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG "instance": "10.142.0.107:9100" 01:05:48 DEBUG "pod": "ds-idrepo-0" 01:05:48 DEBUG "0.0001" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG }, 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG }, 01:05:48 DEBUG }, 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG "value": [ 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG "value": [ 01:05:48 DEBUG "value": [ 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG "0.0004333333333332708" 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG "0" 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG [...] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG } 01:05:48 DEBUG "instance": "10.142.0.43:9100" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "5373.866666666667" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-idrepo-0" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-idrepo-0" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053148" 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "ds-idrepo-1" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-idrepo-0" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "instance": "10.142.0.107:9100" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "instance": "10.142.0.43:9100" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "12117.666666666664" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053148" 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG { 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "ds-idrepo-2" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "6.642723990157797" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG "result": [ 01:05:48 DEBUG { 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG [...] 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "1844879360" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053148" 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG { 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG "data": { 01:05:48 DEBUG "resultType": "vector", 01:05:48 INFO [http_cmd]: http status code OK 01:05:48 DEBUG "result": [ 01:05:48 DEBUG --- status code --- 01:05:48 DEBUG { 01:05:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:05:48 DEBUG "metric": { 01:05:48 DEBUG --- http response --- 01:05:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 01:05:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:05:48 DEBUG 01:05:48 DEBUG ----- output ----- 01:05:48 DEBUG 01:05:48 DEBUG { 01:05:48 DEBUG [...] 01:05:48 DEBUG "status": "success", 01:05:48 DEBUG 01:05:48 DEBUG "data": { 01:05:48 DEBUG 01:05:48 DEBUG "resultType": "vector", 01:05:48 DEBUG 01:05:48 DEBUG "result": [ 01:05:48 DEBUG "pod": "ds-idrepo-2" 01:05:48 DEBUG { 01:05:48 DEBUG }, 01:05:48 DEBUG "metric": { 01:05:48 DEBUG "value": [ 01:05:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG }, 01:05:48 DEBUG "0" 01:05:48 DEBUG "value": [ 01:05:48 DEBUG ] 01:05:48 DEBUG 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG ] 01:05:48 DEBUG 01:05:48 DEBUG } 01:05:48 DEBUG [...] 01:05:48 DEBUG } 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG 01:05:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:05:48 DEBUG }, 01:05:48 DEBUG "value": [ 01:05:48 DEBUG 1713053148, 01:05:48 DEBUG "0" 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG ] 01:05:48 DEBUG } 01:05:48 DEBUG } 01:05:50 INFO 01:05:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:05:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:05:50 INFO [loop_until]: OK (rc = 0) 01:05:50 DEBUG --- stdout --- 01:05:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12415m 8993Mi am-7d597cf7d6-8vr8f 11870m 8393Mi am-7d597cf7d6-9wz9h 15291m 9095Mi ds-cts-0 11509m 7738Mi ds-cts-1 11182m 8010Mi ds-cts-2 11231m 8089Mi ds-idrepo-0 13335m 4276Mi ds-idrepo-1 2024m 4293Mi ds-idrepo-2 1858m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 970Mi idm-b85f9d978-rl76f 5m 973Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6691m 1665Mi 01:05:50 DEBUG --- stderr --- 01:05:50 DEBUG 01:05:51 INFO 01:05:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:05:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:05:51 INFO [loop_until]: OK (rc = 0) 01:05:51 DEBUG --- stdout --- 01:05:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4813Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11589m 38% 10191Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12889m 43% 11049Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17335m 58% 2997Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2029m 6% 6307Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11357m 38% 11181Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 2896Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12353m 41% 9823Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 2007m 6% 5462Mi 4% gke-xlou-cdm-ds-0f601751-j488 11538m 38% 9444Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2210Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16425m 55% 10626Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13201m 44% 5688Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6778m 42% 5422Mi 9% 01:05:51 DEBUG --- stderr --- 01:05:51 DEBUG 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 ERROR [http_cmd]: ERROR 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG --- status code --- 01:06:48 ERROR --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 ERROR --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:06:48 DEBUG { 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:06:48 DEBUG "status": "success", 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053208" 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG "data": { 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "0.012" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "2310.8333333333335" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "0" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG "instance": "10.142.0.107:9100" 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG }, 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG "value": [ 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG [...] 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG "instance": "10.142.0.43:9100" 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG }, 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG }, 01:06:48 DEBUG "0" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG ] 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG } 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG ] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG } 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG } 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:06:48 DEBUG 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG "pod": "ds-idrepo-0" 01:06:48 DEBUG { 01:06:48 DEBUG "pod": "ds-idrepo-0" 01:06:48 DEBUG { 01:06:48 DEBUG }, 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG }, 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "data": { 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "data": { 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "0" 01:06:48 DEBUG "result": [ 01:06:48 DEBUG "0" 01:06:48 DEBUG "result": [ 01:06:48 DEBUG ] 01:06:48 DEBUG { 01:06:48 DEBUG ] 01:06:48 DEBUG { 01:06:48 DEBUG } 01:06:48 DEBUG "metric": { 01:06:48 DEBUG } 01:06:48 DEBUG "metric": { 01:06:48 DEBUG ] 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG ] 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG } 01:06:48 DEBUG }, 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 DEBUG } 01:06:48 DEBUG }, 01:06:48 DEBUG } 01:06:48 DEBUG "value": [ 01:06:48 DEBUG } 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG "pod": "ds-cts-1" 01:06:48 DEBUG }, 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "2304.1666666666665" 01:06:48 DEBUG "0" 01:06:48 DEBUG ] 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "2304.0333333333333" 01:06:48 DEBUG ] 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG } 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG ] 01:06:48 DEBUG { 01:06:48 DEBUG } 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG } 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "2304.3" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "0" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "instance": "10.142.0.107:9100" 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG "instance": "10.142.0.107:9100" 01:06:48 DEBUG }, 01:06:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:06:48 DEBUG }, 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG { 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG [...] 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "data": { 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "result": [ 01:06:48 DEBUG "pod": "ds-cts-1" 01:06:48 DEBUG 01:06:48 DEBUG "instance": "10.142.0.43:9100" 01:06:48 DEBUG "instance": "10.142.0.43:9100" 01:06:48 DEBUG { 01:06:48 DEBUG }, 01:06:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:06:48 DEBUG }, 01:06:48 DEBUG }, 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "value": [ 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "instance": "10.142.0.107:9100" 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG }, 01:06:48 DEBUG "0.00020000000000000004" 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "12599421952" 01:06:48 DEBUG "6839.5" 01:06:48 DEBUG "value": [ 01:06:48 DEBUG ] 01:06:48 DEBUG "414497" 01:06:48 DEBUG ] 01:06:48 DEBUG ] 01:06:48 DEBUG 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG ] 01:06:48 DEBUG 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG [...] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 DEBUG 01:06:48 DEBUG } 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "instance": "10.142.0.43:9100" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "0.000500000000000019" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:06:48 DEBUG "pod": "ds-idrepo-0" 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG }, 01:06:48 DEBUG { 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "data": { 01:06:48 DEBUG "0" 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG ] 01:06:48 DEBUG "result": [ 01:06:48 DEBUG } 01:06:48 DEBUG { 01:06:48 DEBUG ] 01:06:48 DEBUG "metric": { 01:06:48 DEBUG } 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG } 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "ds-idrepo-0" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "0" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "ds-idrepo-1" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "ds-idrepo-0" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "0" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "instance": "10.142.0.107:9100" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "instance": "10.142.0.43:9100" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "12187.799999999997" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "132044284.35955402" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "6.672761518522282" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG { 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG [...] 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG 01:06:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "1846763520" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053208" 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 INFO [http_cmd]: http status code OK 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG --- status code --- 01:06:48 DEBUG { 01:06:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG --- http response --- 01:06:48 DEBUG "data": { 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG "result": [ 01:06:48 DEBUG { 01:06:48 DEBUG "metric": { 01:06:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:06:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:06:48 DEBUG ----- output ----- 01:06:48 DEBUG }, 01:06:48 DEBUG { 01:06:48 DEBUG "value": [ 01:06:48 DEBUG "status": "success", 01:06:48 DEBUG 01:06:48 DEBUG "data": { 01:06:48 DEBUG 01:06:48 DEBUG "resultType": "vector", 01:06:48 DEBUG 01:06:48 DEBUG "result": [ 01:06:48 DEBUG [...] 01:06:48 DEBUG { 01:06:48 DEBUG 01:06:48 DEBUG "metric": { 01:06:48 DEBUG 01:06:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:06:48 DEBUG 01:06:48 DEBUG }, 01:06:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:06:48 DEBUG "value": [ 01:06:48 DEBUG }, 01:06:48 DEBUG 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG 01:06:48 DEBUG "0" 01:06:48 DEBUG [...] 01:06:48 DEBUG ] 01:06:48 DEBUG 01:06:48 DEBUG } 01:06:48 DEBUG 01:06:48 DEBUG ] 01:06:48 DEBUG 01:06:48 DEBUG } 01:06:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:06:48 DEBUG } 01:06:48 DEBUG }, 01:06:48 DEBUG "value": [ 01:06:48 DEBUG 1713053208, 01:06:48 DEBUG "11.033207712861998" 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG ] 01:06:48 DEBUG } 01:06:48 DEBUG } 01:06:50 INFO 01:06:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:06:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:06:50 INFO [loop_until]: OK (rc = 0) 01:06:50 DEBUG --- stdout --- 01:06:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12261m 9012Mi am-7d597cf7d6-8vr8f 11193m 8394Mi am-7d597cf7d6-9wz9h 15540m 9095Mi ds-cts-0 10991m 7722Mi ds-cts-1 11816m 7990Mi ds-cts-2 11710m 8084Mi ds-idrepo-0 13373m 4277Mi ds-idrepo-1 1881m 4293Mi ds-idrepo-2 1933m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 969Mi idm-b85f9d978-rl76f 7m 973Mi lodemon-64cbdcc7b4-vt4kk 10m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6626m 1666Mi 01:06:50 DEBUG --- stderr --- 01:06:50 DEBUG 01:06:51 INFO 01:06:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:06:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:06:51 INFO [loop_until]: OK (rc = 0) 01:06:51 DEBUG --- stdout --- 01:06:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 66m 0% 4802Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11508m 38% 10192Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12532m 41% 11075Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17530m 58% 2990Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2120m 7% 6303Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 10826m 36% 11170Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 2904Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12198m 40% 9830Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1919m 6% 5465Mi 4% gke-xlou-cdm-ds-0f601751-j488 11852m 39% 9434Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2217Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16552m 55% 10619Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13274m 44% 5671Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6853m 43% 5423Mi 9% 01:06:51 DEBUG --- stderr --- 01:06:51 DEBUG 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 ERROR [http_cmd]: ERROR 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG --- status code --- 01:07:48 ERROR --- status code --- 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG { 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG --- http response --- 01:07:48 ERROR --- http response --- 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG "data": { 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:07:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:07:48 DEBUG "result": [ 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG { 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG }, 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG "value": [ 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG }, 01:07:48 DEBUG [...] 01:07:48 DEBUG }, 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG [...] 01:07:48 DEBUG }, 01:07:48 DEBUG [...] 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG "0.012" 01:07:48 DEBUG 01:07:48 DEBUG { 01:07:48 DEBUG { 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG { 01:07:48 DEBUG ] 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG }, 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG } 01:07:48 DEBUG }, 01:07:48 DEBUG "data": { 01:07:48 DEBUG "data": { 01:07:48 DEBUG "value": [ 01:07:48 DEBUG "data": { 01:07:48 DEBUG ] 01:07:48 DEBUG "value": [ 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG } 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG { 01:07:48 DEBUG "2294.0333333333333" 01:07:48 DEBUG "result": [ 01:07:48 DEBUG } 01:07:48 DEBUG "2277.1666666666665" 01:07:48 DEBUG "result": [ 01:07:48 DEBUG "metric": { 01:07:48 DEBUG ] 01:07:48 DEBUG { 01:07:48 DEBUG ] 01:07:48 DEBUG { 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG } 01:07:48 DEBUG "metric": { 01:07:48 DEBUG } 01:07:48 DEBUG "metric": { 01:07:48 DEBUG }, 01:07:48 DEBUG ] 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG ] 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG "value": [ 01:07:48 DEBUG } 01:07:48 DEBUG }, 01:07:48 DEBUG } 01:07:48 DEBUG }, 01:07:48 DEBUG 01:07:48 DEBUG } 01:07:48 DEBUG "value": [ 01:07:48 DEBUG } 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [...] 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG "pod": "ds-cts-1" 01:07:48 DEBUG 01:07:48 DEBUG { 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG 01:07:48 DEBUG { 01:07:48 DEBUG }, 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "value": [ 01:07:48 DEBUG }, 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG "data": { 01:07:48 DEBUG }, 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG "data": { 01:07:48 DEBUG 1713053268, 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG "value": [ 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "value": [ 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "0" 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG { 01:07:48 DEBUG "result": [ 01:07:48 DEBUG ] 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG "0" 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG "data": { 01:07:48 DEBUG { 01:07:48 DEBUG "2288.8333333333335" 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG { 01:07:48 DEBUG } 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG ] 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "metric": { 01:07:48 DEBUG ] 01:07:48 DEBUG "data": { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG "result": [ 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG } 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG }, 01:07:48 DEBUG ] 01:07:48 DEBUG "result": [ 01:07:48 DEBUG }, 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG { 01:07:48 DEBUG "value": [ 01:07:48 DEBUG } 01:07:48 DEBUG { 01:07:48 DEBUG "value": [ 01:07:48 DEBUG } 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG 01:07:48 DEBUG } 01:07:48 DEBUG "metric": { 01:07:48 DEBUG 01:07:48 DEBUG "data": { 01:07:48 DEBUG }, 01:07:48 DEBUG "data": { 01:07:48 DEBUG 01:07:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:07:48 DEBUG 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "value": [ 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG 01:07:48 DEBUG }, 01:07:48 DEBUG 01:07:48 DEBUG "result": [ 01:07:48 DEBUG 01:07:48 DEBUG "result": [ 01:07:48 DEBUG [...] 01:07:48 DEBUG "value": [ 01:07:48 DEBUG [...] 01:07:48 DEBUG { 01:07:48 DEBUG 01:07:48 DEBUG { 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "metric": { 01:07:48 DEBUG 01:07:48 DEBUG "metric": { 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:07:48 DEBUG [...] 01:07:48 DEBUG "instance": "10.142.0.107:9100" 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG }, 01:07:48 DEBUG 01:07:48 DEBUG }, 01:07:48 DEBUG "pod": "ds-idrepo-0" 01:07:48 DEBUG [...] 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG "value": [ 01:07:48 DEBUG }, 01:07:48 DEBUG 01:07:48 DEBUG }, 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-0" 01:07:48 DEBUG 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG 01:07:48 DEBUG }, 01:07:48 DEBUG 01:07:48 DEBUG "0" 01:07:48 DEBUG "pod": "ds-idrepo-2" 01:07:48 DEBUG "414415.5" 01:07:48 DEBUG [...] 01:07:48 DEBUG "value": [ 01:07:48 DEBUG [...] 01:07:48 DEBUG ] 01:07:48 DEBUG }, 01:07:48 DEBUG ] 01:07:48 DEBUG 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG 01:07:48 DEBUG } 01:07:48 DEBUG "value": [ 01:07:48 DEBUG } 01:07:48 DEBUG 01:07:48 DEBUG "0" 01:07:48 DEBUG 01:07:48 DEBUG ] 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG ] 01:07:48 DEBUG 01:07:48 DEBUG ] 01:07:48 DEBUG 01:07:48 DEBUG } 01:07:48 DEBUG "5260.378710949167" 01:07:48 DEBUG } 01:07:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:07:48 DEBUG } 01:07:48 DEBUG "instance": "10.142.0.43:9100" 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG }, 01:07:48 DEBUG ] 01:07:48 DEBUG }, 01:07:48 DEBUG } 01:07:48 DEBUG "value": [ 01:07:48 DEBUG } 01:07:48 DEBUG "value": [ 01:07:48 DEBUG ] 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG } 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG } 01:07:48 DEBUG "2288.133333333333" 01:07:48 DEBUG "0" 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "instance": "10.142.0.107:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "instance": "10.142.0.43:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "12600069120" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-0" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "0" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-cts-1" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "0.00035925" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "instance": "10.142.0.107:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "instance": "10.142.0.43:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "5274.066666666667" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "instance": "10.142.0.107:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "instance": "10.142.0.43:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "0.00040000000000001514" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-0" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "0" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "ds-idrepo-1" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-0" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "0" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-2" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "7095.394540350039" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "instance": "10.142.0.107:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "instance": "10.142.0.43:9100" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "11985.93333333333" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "6.6562487979763265" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "1844289536" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053268" 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-2" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 INFO [http_cmd]: http status code OK 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG --- status code --- 01:07:48 DEBUG "1.901127113075697" 01:07:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:07:48 DEBUG ] 01:07:48 DEBUG --- http response --- 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:07:48 DEBUG ----- output ----- 01:07:48 DEBUG { 01:07:48 DEBUG "status": "success", 01:07:48 DEBUG "data": { 01:07:48 DEBUG "resultType": "vector", 01:07:48 DEBUG "result": [ 01:07:48 DEBUG { 01:07:48 DEBUG "metric": { 01:07:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG [...] 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG 01:07:48 DEBUG "pod": "ds-idrepo-2" 01:07:48 DEBUG }, 01:07:48 DEBUG "value": [ 01:07:48 DEBUG 1713053268, 01:07:48 DEBUG "0" 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG ] 01:07:48 DEBUG } 01:07:48 DEBUG } 01:07:50 INFO 01:07:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:07:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:07:50 INFO [loop_until]: OK (rc = 0) 01:07:50 DEBUG --- stdout --- 01:07:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12421m 8993Mi am-7d597cf7d6-8vr8f 11518m 8392Mi am-7d597cf7d6-9wz9h 14931m 9096Mi ds-cts-0 11290m 7718Mi ds-cts-1 11506m 7998Mi ds-cts-2 11314m 8095Mi ds-idrepo-0 13186m 4277Mi ds-idrepo-1 2008m 4293Mi ds-idrepo-2 1864m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 969Mi idm-b85f9d978-rl76f 6m 973Mi lodemon-64cbdcc7b4-vt4kk 4m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6596m 1668Mi 01:07:50 DEBUG --- stderr --- 01:07:50 DEBUG 01:07:51 INFO 01:07:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:07:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:07:51 INFO [loop_until]: OK (rc = 0) 01:07:51 DEBUG --- stdout --- 01:07:51 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 4804Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11670m 39% 10214Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13015m 43% 11096Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17531m 58% 2995Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2075m 6% 6304Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11208m 37% 11189Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 68m 0% 2892Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12324m 41% 9819Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1975m 6% 5463Mi 4% gke-xlou-cdm-ds-0f601751-j488 11867m 39% 9445Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2200Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16412m 54% 10635Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13252m 44% 5672Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6805m 42% 5419Mi 9% 01:07:51 DEBUG --- stderr --- 01:07:51 DEBUG 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 ERROR [http_cmd]: ERROR 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 ERROR --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 ERROR --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:08:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:08:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:08:48 DEBUG { 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:08:48 DEBUG { 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG { 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG "data": { 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG "data": { 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG "data": { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "data": { 01:08:48 DEBUG { 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "status": "success", 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "status": "success", 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG { 01:08:48 DEBUG "data": { 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG "data": { 01:08:48 DEBUG "data": { 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG { 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "data": { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG "metric": { 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:08:48 DEBUG { 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG "result": [ 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "result": [ 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "metric": { 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG { 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG { 01:08:48 DEBUG }, 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG { 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG }, 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "metric": { 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "value": [ 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG }, 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG }, 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "instance": "10.142.0.107:9100" 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG { 01:08:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:08:48 DEBUG 01:08:48 DEBUG { 01:08:48 DEBUG "data": { 01:08:48 DEBUG "value": [ 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG }, 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG }, 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "data": { 01:08:48 DEBUG "data": { 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "data": { 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG "data": { 01:08:48 DEBUG "result": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG { 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "result": [ 01:08:48 DEBUG 01:08:48 DEBUG "result": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "metric": { 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG { 01:08:48 DEBUG { 01:08:48 DEBUG 01:08:48 DEBUG { 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG { 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG [...] 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG [...] 01:08:48 DEBUG "metric": { 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG "metric": { 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG 01:08:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:08:48 DEBUG 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG "instance": "10.142.0.107:9100" 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "pod": "ds-cts-1" 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "2296.9" 01:08:48 DEBUG ] 01:08:48 DEBUG "0.012" 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG "2290.9" 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG }, 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "instance": "10.142.0.43:9100" 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG ] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "0" 01:08:48 DEBUG }, 01:08:48 DEBUG }, 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG [...] 01:08:48 DEBUG [...] 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG [...] 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG } 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "0" 01:08:48 DEBUG ] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "2294.1666666666665" 01:08:48 DEBUG 01:08:48 DEBUG "3643.730989102992" 01:08:48 DEBUG ] 01:08:48 DEBUG 01:08:48 DEBUG ] 01:08:48 DEBUG "value": [ 01:08:48 DEBUG }, 01:08:48 DEBUG "0" 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG ] 01:08:48 DEBUG 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG 01:08:48 DEBUG } 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG ] 01:08:48 DEBUG "0" 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG } 01:08:48 DEBUG 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG 01:08:48 DEBUG ] 01:08:48 DEBUG "0" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG 01:08:48 DEBUG }, 01:08:48 DEBUG ] 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG ] 01:08:48 DEBUG "instance": "10.142.0.43:9100" 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "0" 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:08:48 DEBUG "value": [ 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG ] 01:08:48 DEBUG ] 01:08:48 DEBUG ] 01:08:48 DEBUG ] 01:08:48 DEBUG }, 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG }, 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "0" 01:08:48 DEBUG "value": [ 01:08:48 DEBUG "12600855552" 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG ] 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG "413768.5" 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053328" 01:08:48 DEBUG "2295.4" 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG ] 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053328" 01:08:48 DEBUG ] 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "instance": "10.142.0.107:9100" 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG }, 01:08:48 DEBUG "instance": "10.142.0.107:9100" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "instance": "10.142.0.43:9100" 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-cts-1" 01:08:48 DEBUG }, 01:08:48 DEBUG "instance": "10.142.0.43:9100" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "0.0003999999999997783" 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "0.0011659722222222223" 01:08:48 DEBUG ] 01:08:48 DEBUG "6635.733333333333" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "instance": "10.142.0.107:9100" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "instance": "10.142.0.43:9100" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "12289.733333333332" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "pod": "ds-idrepo-1" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "0" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "4997.116785055532" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "8.570811558800354" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "1846566912" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "6.538953550615855" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053328" 01:08:48 INFO [http_cmd]: http status code OK 01:08:48 DEBUG --- status code --- 01:08:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:08:48 DEBUG --- http response --- 01:08:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:08:48 DEBUG ----- output ----- 01:08:48 DEBUG { 01:08:48 DEBUG "status": "success", 01:08:48 DEBUG "data": { 01:08:48 DEBUG "resultType": "vector", 01:08:48 DEBUG "result": [ 01:08:48 DEBUG { 01:08:48 DEBUG "metric": { 01:08:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG [...] 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG 01:08:48 DEBUG "pod": "ds-idrepo-0" 01:08:48 DEBUG }, 01:08:48 DEBUG "value": [ 01:08:48 DEBUG 1713053328, 01:08:48 DEBUG "0" 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG ] 01:08:48 DEBUG } 01:08:48 DEBUG } 01:08:50 INFO 01:08:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:08:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:08:50 INFO [loop_until]: OK (rc = 0) 01:08:50 DEBUG --- stdout --- 01:08:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12566m 8998Mi am-7d597cf7d6-8vr8f 11350m 8392Mi am-7d597cf7d6-9wz9h 15065m 9098Mi ds-cts-0 11282m 7729Mi ds-cts-1 11942m 7996Mi ds-cts-2 11568m 8074Mi ds-idrepo-0 13126m 4277Mi ds-idrepo-1 2007m 4293Mi ds-idrepo-2 1889m 4114Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 9m 970Mi idm-b85f9d978-rl76f 7m 973Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6499m 1670Mi 01:08:50 DEBUG --- stderr --- 01:08:50 DEBUG 01:08:51 INFO 01:08:51 INFO [loop_until]: kubectl --namespace=xlou top node 01:08:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:08:52 INFO [loop_until]: OK (rc = 0) 01:08:52 DEBUG --- stdout --- 01:08:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4804Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11582m 38% 10219Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 13063m 43% 11066Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 16869m 56% 2992Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2110m 7% 6296Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11465m 38% 11188Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 71m 0% 2904Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12234m 40% 9832Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1974m 6% 5461Mi 4% gke-xlou-cdm-ds-0f601751-j488 11795m 39% 9454Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2206Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16410m 54% 10641Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13344m 44% 5656Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6693m 42% 5421Mi 9% 01:08:52 DEBUG --- stderr --- 01:08:52 DEBUG 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053388" 01:09:48 DEBUG --- status code --- 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053388" 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 DEBUG ----- output ----- 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG { 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 DEBUG { 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- status code --- 01:09:48 ERROR [http_cmd]: ERROR 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG --- status code --- 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG "data": { 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 ERROR --- status code --- 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG "data": { 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG "result": [ 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:09:48 ERROR --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG "result": [ 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:09:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:09:48 DEBUG { 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG "metric": { 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "metric": { 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG { 01:09:48 DEBUG "instance": "10.142.0.107:9100" 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:09:48 DEBUG "data": { 01:09:48 DEBUG "data": { 01:09:48 DEBUG { 01:09:48 DEBUG }, 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG "data": { 01:09:48 DEBUG "data": { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG }, 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "data": { 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG { 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "data": { 01:09:48 DEBUG 01:09:48 DEBUG "result": [ 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG 01:09:48 DEBUG "resultType": "vector", 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG "result": [ 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG "result": [ 01:09:48 DEBUG 01:09:48 DEBUG "result": [ 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG "data": { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "result": [ 01:09:48 DEBUG 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "metric": { 01:09:48 DEBUG { 01:09:48 DEBUG 01:09:48 DEBUG { 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG { 01:09:48 DEBUG [...] 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 DEBUG "data": { 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 DEBUG "metric": { 01:09:48 DEBUG [...] 01:09:48 DEBUG "metric": { 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG "instance": "10.142.0.107:9100" 01:09:48 DEBUG "data": { 01:09:48 DEBUG "data": { 01:09:48 DEBUG "data": { 01:09:48 DEBUG "result": [ 01:09:48 DEBUG }, 01:09:48 DEBUG }, 01:09:48 DEBUG "metric": { 01:09:48 DEBUG 01:09:48 DEBUG }, 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG }, 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 DEBUG 01:09:48 DEBUG "instance": "10.142.0.107:9100" 01:09:48 DEBUG }, 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "value": [ 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "result": [ 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "value": [ 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG { 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053388" 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG { 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "metric": { 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "instance": "10.142.0.43:9100" 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "instance": "10.142.0.107:9100" 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "data": { 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG [...] 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG }, 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG }, 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG "result": [ 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG "value": [ 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG "12601390080" 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG { 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-idrepo-0" 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-idrepo-0" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG }, 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG }, 01:09:48 DEBUG 01:09:48 DEBUG "metric": { 01:09:48 DEBUG } 01:09:48 DEBUG "instance": "10.142.0.43:9100" 01:09:48 DEBUG 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-idrepo-0" 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG } 01:09:48 DEBUG }, 01:09:48 DEBUG [...] 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG [...] 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG 01:09:48 DEBUG }, 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "pod": "ds-cts-1" 01:09:48 DEBUG "0" 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "instance": "10.142.0.43:9100" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG "data": { 01:09:48 DEBUG "2303.8" 01:09:48 DEBUG 01:09:48 DEBUG "result": [ 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "0" 01:09:48 DEBUG }, 01:09:48 DEBUG ] 01:09:48 DEBUG "pod": "ds-cts-1" 01:09:48 DEBUG 01:09:48 DEBUG "0" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG { 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG ] 01:09:48 DEBUG "value": [ 01:09:48 DEBUG } 01:09:48 DEBUG }, 01:09:48 DEBUG "instance": "10.142.0.43:9100" 01:09:48 DEBUG ] 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG 01:09:48 DEBUG "metric": { 01:09:48 DEBUG }, 01:09:48 DEBUG "0" 01:09:48 DEBUG } 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG "value": [ 01:09:48 DEBUG } 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "0.00033333333333333365" 01:09:48 DEBUG 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG "pod": "ds-idrepo-0" 01:09:48 DEBUG }, 01:09:48 DEBUG ] 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG ] 01:09:48 DEBUG "0" 01:09:48 DEBUG }, 01:09:48 DEBUG } 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG ] 01:09:48 DEBUG "5771.366666666667" 01:09:48 DEBUG ] 01:09:48 DEBUG [...] 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG } 01:09:48 DEBUG }, 01:09:48 DEBUG }, 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "0.00013333333333333334" 01:09:48 DEBUG ] 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG "pod": "ds-cts-2" 01:09:48 DEBUG 01:09:48 DEBUG "0" 01:09:48 DEBUG } 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "2293.133333333333" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG 01:09:48 DEBUG }, 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG ] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG "12034.766666666665" 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG } 01:09:48 DEBUG }, 01:09:48 DEBUG "data": { 01:09:48 DEBUG ] 01:09:48 DEBUG [...] 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG }, 01:09:48 DEBUG "46404139.920353025" 01:09:48 DEBUG ] 01:09:48 DEBUG "value": [ 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG "value": [ 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "result": [ 01:09:48 DEBUG ] 01:09:48 DEBUG 01:09:48 DEBUG ] 01:09:48 DEBUG 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG "0.012" 01:09:48 DEBUG { 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG "2293.3" 01:09:48 DEBUG ] 01:09:48 DEBUG ] 01:09:48 DEBUG "metric": { 01:09:48 DEBUG } 01:09:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:09:48 DEBUG } 01:09:48 DEBUG }, 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG "instance": "10.142.0.107:9100" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "2293.0666666666666" 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG "412959.5" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG [...] 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG } 01:09:48 DEBUG 01:09:48 DEBUG "instance": "10.142.0.43:9100" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "0.0004666666666667633" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "pod": "ds-idrepo-1" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-idrepo-0" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "0" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-idrepo-2" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "0" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-idrepo-2" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "0" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "1846284288" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "6.529105634994228" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053388" 01:09:48 INFO [http_cmd]: http status code OK 01:09:48 DEBUG --- status code --- 01:09:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:09:48 DEBUG --- http response --- 01:09:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:09:48 DEBUG ----- output ----- 01:09:48 DEBUG { 01:09:48 DEBUG "status": "success", 01:09:48 DEBUG "data": { 01:09:48 DEBUG "resultType": "vector", 01:09:48 DEBUG "result": [ 01:09:48 DEBUG { 01:09:48 DEBUG "metric": { 01:09:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG [...] 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG 01:09:48 DEBUG "pod": "ds-cts-2" 01:09:48 DEBUG }, 01:09:48 DEBUG "value": [ 01:09:48 DEBUG 1713053388, 01:09:48 DEBUG "35878832.13145338" 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG ] 01:09:48 DEBUG } 01:09:48 DEBUG } 01:09:50 INFO 01:09:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:09:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:09:50 INFO [loop_until]: OK (rc = 0) 01:09:50 DEBUG --- stdout --- 01:09:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12449m 8996Mi am-7d597cf7d6-8vr8f 11641m 8400Mi am-7d597cf7d6-9wz9h 15108m 9095Mi ds-cts-0 11508m 7726Mi ds-cts-1 11679m 8015Mi ds-cts-2 11451m 8078Mi ds-idrepo-0 13283m 4277Mi ds-idrepo-1 1973m 4293Mi ds-idrepo-2 1873m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 970Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 2m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6436m 1671Mi 01:09:50 DEBUG --- stderr --- 01:09:50 DEBUG 01:09:52 INFO 01:09:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:09:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:09:52 INFO [loop_until]: OK (rc = 0) 01:09:52 DEBUG --- stdout --- 01:09:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4802Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11628m 38% 10228Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12761m 42% 11056Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17355m 58% 2997Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2059m 6% 6296Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11479m 38% 11194Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 64m 0% 2901Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12315m 41% 9836Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1984m 6% 5460Mi 4% gke-xlou-cdm-ds-0f601751-j488 11811m 39% 9465Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2201Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16446m 55% 10630Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13218m 44% 5681Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6684m 42% 5418Mi 9% 01:09:52 DEBUG --- stderr --- 01:09:52 DEBUG 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053448" 01:10:48 ERROR [http_cmd]: ERROR 01:10:48 ERROR --- status code --- 01:10:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:10:48 ERROR --- http response --- 01:10:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:10:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053448" 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- status code --- 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053448" 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- status code --- 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG { 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 DEBUG { 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG "data": { 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053448" 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG { 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 DEBUG "data": { 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "data": { 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "resultType": "vector", 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053448" 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG { 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG { 01:10:48 DEBUG "result": [ 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG { 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "data": { 01:10:48 DEBUG { 01:10:48 DEBUG "result": [ 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG { 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG { 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "data": { 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "data": { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "data": { 01:10:48 DEBUG { 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG "result": [ 01:10:48 DEBUG "data": { 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG "metric": { 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG }, 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG "result": [ 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG "metric": { 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:10:48 DEBUG "result": [ 01:10:48 DEBUG "data": { 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "value": [ 01:10:48 DEBUG }, 01:10:48 DEBUG { 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG "data": { 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG }, 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG { 01:10:48 DEBUG }, 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "result": [ 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG }, 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "data": { 01:10:48 DEBUG "value": [ 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG { 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "result": [ 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "value": [ 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "instance": "10.142.0.107:9100" 01:10:48 DEBUG 01:10:48 DEBUG "result": [ 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG "data": { 01:10:48 DEBUG 01:10:48 DEBUG "data": { 01:10:48 DEBUG }, 01:10:48 DEBUG "metric": { 01:10:48 DEBUG { 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG { 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "value": [ 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "instance": "10.142.0.107:9100" 01:10:48 DEBUG "metric": { 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "value": [ 01:10:48 DEBUG [...] 01:10:48 DEBUG "metric": { 01:10:48 DEBUG 01:10:48 DEBUG "result": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 DEBUG "result": [ 01:10:48 DEBUG 01:10:48 DEBUG "result": [ 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG "instance": "10.142.0.107:9100" 01:10:48 DEBUG { 01:10:48 DEBUG "data": { 01:10:48 DEBUG 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG [...] 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG }, 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG "result": [ 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG [...] 01:10:48 DEBUG "metric": { 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG { 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG 01:10:48 DEBUG "metric": { 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "data": { 01:10:48 DEBUG "result": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG [...] 01:10:48 DEBUG "pod": "ds-idrepo-0" 01:10:48 DEBUG 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-1" 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG "data": { 01:10:48 DEBUG "data": { 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "result": [ 01:10:48 DEBUG "metric": { 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG "instance": "10.142.0.107:9100" 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "pod": "ds-idrepo-0" 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG { 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG "pod": "ds-cts-1" 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-2" 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG }, 01:10:48 DEBUG "result": [ 01:10:48 DEBUG "result": [ 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG "metric": { 01:10:48 DEBUG }, 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG [...] 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG "0" 01:10:48 DEBUG "value": [ 01:10:48 DEBUG { 01:10:48 DEBUG { 01:10:48 DEBUG 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG "metric": { 01:10:48 DEBUG 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "instance": "10.142.0.43:9100" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "metric": { 01:10:48 DEBUG } 01:10:48 DEBUG "instance": "10.142.0.107:9100" 01:10:48 DEBUG }, 01:10:48 DEBUG "0" 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "instance": "10.142.0.43:9100" 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG "0" 01:10:48 DEBUG "2292.4333333333334" 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "2288.9666666666667" 01:10:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:10:48 DEBUG } 01:10:48 DEBUG [...] 01:10:48 DEBUG "value": [ 01:10:48 DEBUG ] 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG ] 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "5301.611759992604" 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-cts-1" 01:10:48 DEBUG 01:10:48 DEBUG "2288.7666666666664" 01:10:48 DEBUG ] 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "instance": "10.142.0.43:9100" 01:10:48 DEBUG "pod": "ds-idrepo-0" 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "5398.966666666666" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG "0" 01:10:48 DEBUG ] 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG ] 01:10:48 DEBUG "0" 01:10:48 DEBUG }, 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG "instance": "10.142.0.43:9100" 01:10:48 DEBUG 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG ] 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG }, 01:10:48 DEBUG "pod": "ds-idrepo-0" 01:10:48 DEBUG "0.00022220833333333332" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-0" 01:10:48 DEBUG [...] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG "value": [ 01:10:48 DEBUG }, 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG 01:10:48 DEBUG }, 01:10:48 DEBUG 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 DEBUG ] 01:10:48 DEBUG "12602072064" 01:10:48 DEBUG "0" 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG } 01:10:48 DEBUG [...] 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG ] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG "11824.33333333333" 01:10:48 DEBUG } 01:10:48 DEBUG "0" 01:10:48 DEBUG 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG ] 01:10:48 DEBUG ] 01:10:48 DEBUG 01:10:48 DEBUG "0" 01:10:48 DEBUG "instance": "10.142.0.43:9100" 01:10:48 DEBUG ] 01:10:48 DEBUG ] 01:10:48 DEBUG }, 01:10:48 DEBUG }, 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG 01:10:48 DEBUG ] 01:10:48 DEBUG }, 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG "value": [ 01:10:48 DEBUG "value": [ 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:10:48 DEBUG } 01:10:48 DEBUG "value": [ 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG }, 01:10:48 DEBUG ] 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "2289.9" 01:10:48 DEBUG "0.012" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG "value": [ 01:10:48 DEBUG } 01:10:48 DEBUG "0.00043333333333350765" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG "412640.5" 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-2" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "11234.367777127185" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053448" 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "1849344000" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-2" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "1.885394711701453" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "6.55762561874413" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053448" 01:10:48 INFO [http_cmd]: http status code OK 01:10:48 DEBUG --- status code --- 01:10:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:10:48 DEBUG --- http response --- 01:10:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:10:48 DEBUG ----- output ----- 01:10:48 DEBUG { 01:10:48 DEBUG "status": "success", 01:10:48 DEBUG "data": { 01:10:48 DEBUG "resultType": "vector", 01:10:48 DEBUG "result": [ 01:10:48 DEBUG { 01:10:48 DEBUG "metric": { 01:10:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG [...] 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG 01:10:48 DEBUG "pod": "ds-idrepo-2" 01:10:48 DEBUG }, 01:10:48 DEBUG "value": [ 01:10:48 DEBUG 1713053448, 01:10:48 DEBUG "0" 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG ] 01:10:48 DEBUG } 01:10:48 DEBUG } 01:10:50 INFO 01:10:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:10:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:10:50 INFO [loop_until]: OK (rc = 0) 01:10:50 DEBUG --- stdout --- 01:10:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12411m 8998Mi am-7d597cf7d6-8vr8f 11544m 8399Mi am-7d597cf7d6-9wz9h 15408m 9098Mi ds-cts-0 11186m 7727Mi ds-cts-1 11750m 8002Mi ds-cts-2 11339m 8113Mi ds-idrepo-0 13015m 4277Mi ds-idrepo-1 1954m 4293Mi ds-idrepo-2 1854m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 970Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 9m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6405m 1673Mi 01:10:50 DEBUG --- stderr --- 01:10:50 DEBUG 01:10:52 INFO 01:10:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:10:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:10:52 INFO [loop_until]: OK (rc = 0) 01:10:52 DEBUG --- stdout --- 01:10:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4805Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11546m 38% 10237Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12958m 43% 11066Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17459m 58% 2981Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2142m 7% 6307Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11356m 38% 11175Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 71m 0% 2910Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 11850m 39% 9831Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1957m 6% 5461Mi 4% gke-xlou-cdm-ds-0f601751-j488 11848m 39% 9450Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2196Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 15893m 53% 10628Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13274m 44% 5664Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6702m 42% 5419Mi 9% 01:10:52 DEBUG --- stderr --- 01:10:52 DEBUG 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 ERROR [http_cmd]: ERROR 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 ERROR --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 ERROR --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:11:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:11:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053508" 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG ----- output ----- 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:11:48 DEBUG { 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG { 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG { 01:11:48 DEBUG "data": { 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "data": { 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG { 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG { 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "data": { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG { 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG "data": { 01:11:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:11:48 DEBUG "instance": "10.142.0.107:9100" 01:11:48 DEBUG "data": { 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "instance": "10.142.0.107:9100" 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG "instance": "10.142.0.107:9100" 01:11:48 DEBUG "instance": "10.142.0.107:9100" 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG "metric": { 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG }, 01:11:48 DEBUG "instance": "10.142.0.107:9100" 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "result": [ 01:11:48 DEBUG { 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG { 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "pod": "ds-idrepo-1" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG [...] 01:11:48 DEBUG [...] 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG [...] 01:11:48 DEBUG [...] 01:11:48 DEBUG [...] 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "pod": "ds-idrepo-0" 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "instance": "10.142.0.43:9100" 01:11:48 DEBUG "pod": "ds-cts-1" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "pod": "ds-idrepo-0" 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG }, 01:11:48 DEBUG "instance": "10.142.0.43:9100" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "instance": "10.142.0.43:9100" 01:11:48 DEBUG "pod": "ds-idrepo-0" 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG "instance": "10.142.0.43:9100" 01:11:48 DEBUG [...] 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG "instance": "10.142.0.43:9100" 01:11:48 DEBUG [...] 01:11:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG "0" 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "pod": "ds-idrepo-0" 01:11:48 DEBUG 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "2296.366666666667" 01:11:48 DEBUG ] 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG }, 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG ] 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG }, 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG } 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG }, 01:11:48 DEBUG "6.521510893569885" 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 01:11:48 DEBUG "11894.066666666664" 01:11:48 DEBUG "0.00006666666666666667" 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG }, 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG "0" 01:11:48 DEBUG }, 01:11:48 DEBUG "412229" 01:11:48 DEBUG } 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG ] 01:11:48 DEBUG }, 01:11:48 DEBUG "pod": "ds-cts-1" 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG } 01:11:48 DEBUG "6824.433333333333" 01:11:48 DEBUG "0.0125" 01:11:48 DEBUG "0.00033333333333333365" 01:11:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG "0.000333333333333267" 01:11:48 DEBUG "pod": "ds-idrepo-0" 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "value": [ 01:11:48 DEBUG ] 01:11:48 DEBUG "value": [ 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG "0" 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG } 01:11:48 DEBUG "value": [ 01:11:48 DEBUG }, 01:11:48 DEBUG "0" 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG }, 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG "1853235200" 01:11:48 DEBUG }, 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "0" 01:11:48 DEBUG } 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "12602686464" 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG "value": [ 01:11:48 DEBUG "181736.99744211975" 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG "0" 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG "12.44056716075308" 01:11:48 DEBUG ] 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "2283.4333333333334" 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "2283.366666666667" 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG "2283.5666666666666" 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG "4193248.7702498846" 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG "0" 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053508" 01:11:48 INFO [http_cmd]: http status code OK 01:11:48 DEBUG --- status code --- 01:11:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:11:48 DEBUG --- http response --- 01:11:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:11:48 DEBUG ----- output ----- 01:11:48 DEBUG { 01:11:48 DEBUG "status": "success", 01:11:48 DEBUG "data": { 01:11:48 DEBUG "resultType": "vector", 01:11:48 DEBUG "result": [ 01:11:48 DEBUG { 01:11:48 DEBUG "metric": { 01:11:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG [...] 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG 01:11:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:11:48 DEBUG }, 01:11:48 DEBUG "value": [ 01:11:48 DEBUG 1713053508, 01:11:48 DEBUG "0" 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG ] 01:11:48 DEBUG } 01:11:48 DEBUG } 01:11:50 INFO 01:11:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:11:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:11:50 INFO [loop_until]: OK (rc = 0) 01:11:50 DEBUG --- stdout --- 01:11:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12287m 9001Mi am-7d597cf7d6-8vr8f 11516m 8398Mi am-7d597cf7d6-9wz9h 15607m 9115Mi ds-cts-0 11266m 7738Mi ds-cts-1 11643m 8009Mi ds-cts-2 11363m 8089Mi ds-idrepo-0 12906m 4277Mi ds-idrepo-1 2009m 4293Mi ds-idrepo-2 1878m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 970Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 9m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6493m 1674Mi 01:11:50 DEBUG --- stderr --- 01:11:50 DEBUG 01:11:52 INFO 01:11:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:11:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:11:52 INFO [loop_until]: OK (rc = 0) 01:11:52 DEBUG --- stdout --- 01:11:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4811Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11511m 38% 10220Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12853m 43% 11056Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17409m 58% 2986Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2074m 6% 6303Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11414m 38% 11171Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2898Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12255m 41% 9834Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1990m 6% 5462Mi 4% gke-xlou-cdm-ds-0f601751-j488 11797m 39% 9467Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2186Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16519m 55% 10643Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13253m 44% 5675Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6482m 40% 5419Mi 9% 01:11:52 DEBUG --- stderr --- 01:11:52 DEBUG 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 ERROR [http_cmd]: ERROR 01:12:48 ERROR --- status code --- 01:12:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:12:48 ERROR --- http response --- 01:12:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:12:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053568" 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "ds-idrepo-0" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "0" 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "411967.5" 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053568" 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "0.013" 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053568" 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 DEBUG --- status code --- 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053568" 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG --- http response --- 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG { 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG { 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG { 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG { 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG "data": { 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG "data": { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "data": { 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "data": { 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG "data": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:12:48 DEBUG "data": { 01:12:48 DEBUG "data": { 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG { 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:12:48 DEBUG "result": [ 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "result": [ 01:12:48 INFO [http_cmd]: http status code OK 01:12:48 DEBUG --- status code --- 01:12:48 DEBUG { 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG { 01:12:48 DEBUG "result": [ 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG { 01:12:48 DEBUG "data": { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "result": [ 01:12:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:12:48 DEBUG "metric": { 01:12:48 DEBUG { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG { 01:12:48 DEBUG --- http response --- 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "instance": "10.142.0.107:9100" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "instance": "10.142.0.107:9100" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "data": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG { 01:12:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:12:48 DEBUG }, 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG }, 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG "result": [ 01:12:48 DEBUG }, 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG "instance": "10.142.0.107:9100" 01:12:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:12:48 DEBUG "instance": "10.142.0.107:9100" 01:12:48 DEBUG "result": [ 01:12:48 DEBUG { 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG }, 01:12:48 DEBUG ----- output ----- 01:12:48 DEBUG "value": [ 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG }, 01:12:48 DEBUG { 01:12:48 DEBUG "value": [ 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG "result": [ 01:12:48 DEBUG "result": [ 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG "metric": { 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG "instance": "10.142.0.107:9100" 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG { 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "metric": { 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG { 01:12:48 DEBUG { 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "status": "success", 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "metric": { 01:12:48 DEBUG "metric": { 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "ds-idrepo-1" 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG "data": { 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:12:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "resultType": "vector", 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "result": [ 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG [...] 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "value": [ 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG [...] 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG { 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG "metric": { 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG "pod": "ds-cts-2" 01:12:48 DEBUG }, 01:12:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:12:48 DEBUG "pod": "ds-idrepo-0" 01:12:48 DEBUG 01:12:48 DEBUG "instance": "10.142.0.43:9100" 01:12:48 DEBUG 01:12:48 DEBUG "pod": "ds-cts-1" 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "pod": "ds-idrepo-0" 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "instance": "10.142.0.43:9100" 01:12:48 DEBUG 01:12:48 DEBUG "instance": "10.142.0.43:9100" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "pod": "ds-idrepo-0" 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG }, 01:12:48 DEBUG }, 01:12:48 DEBUG "pod": "ds-idrepo-0" 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG [...] 01:12:48 DEBUG [...] 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "0.00040000000000001514" 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "instance": "10.142.0.43:9100" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG 01:12:48 DEBUG ] 01:12:48 DEBUG "pod": "ds-cts-1" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG "instance": "10.142.0.43:9100" 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "2279.4666666666667" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG }, 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG }, 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG }, 01:12:48 DEBUG 01:12:48 DEBUG "12603304960" 01:12:48 DEBUG 01:12:48 DEBUG } 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "pod": "ds-cts-2" 01:12:48 DEBUG }, 01:12:48 DEBUG "0" 01:12:48 DEBUG ] 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "0.00033333333333333365" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "0.00006666666666666667" 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "0" 01:12:48 DEBUG 01:12:48 DEBUG 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG ] 01:12:48 DEBUG 01:12:48 DEBUG ] 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG "0" 01:12:48 DEBUG [...] 01:12:48 DEBUG "0" 01:12:48 DEBUG ] 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:12:48 DEBUG }, 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG }, 01:12:48 DEBUG } 01:12:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:12:48 DEBUG } 01:12:48 DEBUG "0" 01:12:48 DEBUG "0" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG } 01:12:48 DEBUG "11623.83333333333" 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG "5198.166666666666" 01:12:48 DEBUG } 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "0" 01:12:48 DEBUG } 01:12:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "6.559413611171757" 01:12:48 DEBUG "value": [ 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG "2279.7" 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG "2279.1" 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG }, 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG }, 01:12:48 DEBUG "value": [ 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG "0" 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG "value": [ 01:12:48 DEBUG "1856004096" 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG "2293.4" 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG }, 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG "0" 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG "value": [ 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG 1713053568, 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG "0" 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:48 DEBUG ] 01:12:48 DEBUG } 01:12:48 DEBUG } 01:12:50 INFO 01:12:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:12:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:12:50 INFO [loop_until]: OK (rc = 0) 01:12:50 DEBUG --- stdout --- 01:12:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12444m 9053Mi am-7d597cf7d6-8vr8f 11412m 8400Mi am-7d597cf7d6-9wz9h 15459m 9109Mi ds-cts-0 11079m 7751Mi ds-cts-1 11608m 8016Mi ds-cts-2 11512m 8087Mi ds-idrepo-0 13085m 4277Mi ds-idrepo-1 2013m 4293Mi ds-idrepo-2 1928m 4114Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 6m 970Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 3m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6571m 1676Mi 01:12:50 DEBUG --- stderr --- 01:12:50 DEBUG 01:12:52 INFO 01:12:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:12:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:12:52 INFO [loop_until]: OK (rc = 0) 01:12:52 DEBUG --- stdout --- 01:12:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4810Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11487m 38% 10235Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12844m 43% 11060Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 16797m 56% 3015Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2089m 6% 6306Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11383m 38% 11191Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2905Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12301m 41% 9834Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1945m 6% 5469Mi 4% gke-xlou-cdm-ds-0f601751-j488 11604m 38% 9455Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2203Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16409m 54% 10632Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13168m 44% 5668Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6676m 42% 5421Mi 9% 01:12:52 DEBUG --- stderr --- 01:12:52 DEBUG 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 ERROR [http_cmd]: ERROR 01:13:48 ERROR --- status code --- 01:13:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:13:48 ERROR --- http response --- 01:13:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:13:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "0" 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "411568.5" 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 DEBUG { 01:13:48 DEBUG --- http response --- 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG "data": { 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG "result": [ 01:13:48 DEBUG }, 01:13:48 DEBUG { 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "metric": { 01:13:48 DEBUG 01:13:48 DEBUG "instance": "10.142.0.107:9100" 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "instance": "10.142.0.43:9100" 01:13:48 DEBUG "0.013" 01:13:48 DEBUG }, 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG ] 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG "value": [ 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG } 01:13:48 DEBUG "0.0004666666666667633" 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG } 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG ] 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG { 01:13:48 DEBUG } 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG { 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG } 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG "data": { 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:13:48 DEBUG "data": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "result": [ 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG "result": [ 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG { 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG "metric": { 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG "instance": "10.142.0.107:9100" 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:13:48 DEBUG }, 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG }, 01:13:48 DEBUG { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "value": [ 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG { 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "data": { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "metric": { 01:13:48 DEBUG 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG { 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG 01:13:48 DEBUG { 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "metric": { 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG { 01:13:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG [...] 01:13:48 DEBUG "data": { 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "metric": { 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG { 01:13:48 DEBUG }, 01:13:48 DEBUG "instance": "10.142.0.43:9100" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG "6.516487372524944" 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG ] 01:13:48 DEBUG "12604109824" 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG "0" 01:13:48 DEBUG 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG ] 01:13:48 DEBUG 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG 01:13:48 DEBUG "pod": "ds-idrepo-2" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG } 01:13:48 DEBUG 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG ] 01:13:48 DEBUG [...] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "0" 01:13:48 DEBUG } 01:13:48 DEBUG 01:13:48 DEBUG } 01:13:48 DEBUG }, 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "0" 01:13:48 DEBUG } 01:13:48 DEBUG 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG ] 01:13:48 DEBUG ] 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG "2280.6" 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG }, 01:13:48 DEBUG ] 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG "value": [ 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG "0" 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 DEBUG --- status code --- 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053628" 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053628" 01:13:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053628" 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG --- http response --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG --- status code --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG { 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG { 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG { 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:13:48 DEBUG --- http response --- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG "data": { 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 DEBUG ----- output ----- 01:13:48 INFO [http_cmd]: http status code OK 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG { 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG { 01:13:48 DEBUG --- status code --- 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG { 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "data": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:13:48 DEBUG { 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG "data": { 01:13:48 DEBUG --- http response --- 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "data": { 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "data": { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "data": { 01:13:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "metric": { 01:13:48 DEBUG { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "instance": "10.142.0.107:9100" 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG ----- output ----- 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG }, 01:13:48 DEBUG { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:13:48 DEBUG }, 01:13:48 DEBUG { 01:13:48 DEBUG "result": [ 01:13:48 DEBUG { 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "metric": { 01:13:48 DEBUG { 01:13:48 DEBUG { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG }, 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "metric": { 01:13:48 DEBUG { 01:13:48 DEBUG "status": "success", 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "data": { 01:13:48 DEBUG 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG "metric": { 01:13:48 DEBUG "data": { 01:13:48 DEBUG "instance": "10.142.0.107:9100" 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG "instance": "10.142.0.107:9100" 01:13:48 DEBUG "pod": "ds-idrepo-1" 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:13:48 DEBUG "resultType": "vector", 01:13:48 DEBUG }, 01:13:48 DEBUG "result": [ 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG "result": [ 01:13:48 DEBUG "value": [ 01:13:48 DEBUG { 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG { 01:13:48 DEBUG 01:13:48 DEBUG "metric": { 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "metric": { 01:13:48 DEBUG 01:13:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG [...] 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG [...] 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "instance": "10.142.0.43:9100" 01:13:48 DEBUG 01:13:48 DEBUG [...] 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG 01:13:48 DEBUG "pod": "ds-idrepo-2" 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG }, 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "pod": "ds-cts-1" 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG 01:13:48 DEBUG 01:13:48 DEBUG "instance": "10.142.0.43:9100" 01:13:48 DEBUG [...] 01:13:48 DEBUG "2286.0666666666666" 01:13:48 DEBUG }, 01:13:48 DEBUG "instance": "10.142.0.43:9100" 01:13:48 DEBUG "pod": "ds-cts-1" 01:13:48 DEBUG }, 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "6926.966666666666" 01:13:48 DEBUG }, 01:13:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:13:48 DEBUG [...] 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG ] 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG }, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "2279.8333333333335" 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "5097.788208520019" 01:13:48 DEBUG ] 01:13:48 DEBUG "value": [ 01:13:48 DEBUG }, 01:13:48 DEBUG 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG } 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG ] 01:13:48 DEBUG "0" 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG 01:13:48 DEBUG ] 01:13:48 DEBUG "0.0001" 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "2279.5666666666666" 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG "0" 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG 01:13:48 DEBUG "0" 01:13:48 DEBUG "pod": "ds-idrepo-0" 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG "12196.83333333333" 01:13:48 DEBUG "0" 01:13:48 DEBUG ] 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG "0" 01:13:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:13:48 DEBUG ] 01:13:48 DEBUG }, 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG }, 01:13:48 DEBUG } 01:13:48 DEBUG "value": [ 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG "value": [ 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG 1713053628, 01:13:48 DEBUG "0" 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG "1857814528" 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG ] 01:13:48 DEBUG ] 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:48 DEBUG } 01:13:50 INFO 01:13:50 INFO [loop_until]: kubectl --namespace=xlou top pods 01:13:50 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:13:50 INFO [loop_until]: OK (rc = 0) 01:13:50 DEBUG --- stdout --- 01:13:50 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12333m 9043Mi am-7d597cf7d6-8vr8f 11379m 8399Mi am-7d597cf7d6-9wz9h 14852m 9093Mi ds-cts-0 11033m 7738Mi ds-cts-1 11796m 8011Mi ds-cts-2 11710m 8092Mi ds-idrepo-0 13277m 4277Mi ds-idrepo-1 2011m 4293Mi ds-idrepo-2 1868m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 970Mi idm-b85f9d978-rl76f 5m 973Mi lodemon-64cbdcc7b4-vt4kk 8m 64Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6491m 1678Mi 01:13:50 DEBUG --- stderr --- 01:13:50 DEBUG 01:13:52 INFO 01:13:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:13:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:13:52 INFO [loop_until]: OK (rc = 0) 01:13:52 DEBUG --- stdout --- 01:13:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 61m 0% 4807Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11489m 38% 10236Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12870m 43% 11115Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17380m 58% 3004Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2071m 6% 6305Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11221m 37% 11219Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 63m 0% 2903Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12221m 40% 9834Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 2007m 6% 5481Mi 4% gke-xlou-cdm-ds-0f601751-j488 11792m 39% 9485Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 65m 0% 2206Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16218m 54% 10636Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13188m 44% 5666Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6659m 41% 5422Mi 9% 01:13:52 DEBUG --- stderr --- 01:13:52 DEBUG 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 ERROR [http_cmd]: ERROR 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 ERROR --- status code --- 01:14:48 DEBUG --- status code --- 01:14:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 ERROR --- http response --- 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:14:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:14:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-idrepo-0" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0.013" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG { 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "411466.5" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "6.525419006090684" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "instance": "10.142.0.107:9100" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "instance": "10.142.0.43:9100" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0.00040000000000001514" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-idrepo-0" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG --- http response --- 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053688" 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG "data": { 01:14:48 DEBUG "data": { 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG "result": [ 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "data": { 01:14:48 DEBUG "data": { 01:14:48 DEBUG "instance": "10.142.0.107:9100" 01:14:48 DEBUG "data": { 01:14:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG }, 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG }, 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG 01:14:48 DEBUG { 01:14:48 DEBUG 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG 01:14:48 DEBUG "metric": { 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:14:48 DEBUG 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG [...] 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG }, 01:14:48 DEBUG [...] 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "instance": "10.142.0.43:9100" 01:14:48 DEBUG 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:14:48 DEBUG [...] 01:14:48 DEBUG [...] 01:14:48 DEBUG }, 01:14:48 DEBUG [...] 01:14:48 DEBUG { 01:14:48 DEBUG }, 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG 01:14:48 DEBUG "data": { 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "12604697600" 01:14:48 DEBUG 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "126209726.12988994" 01:14:48 DEBUG "pod": "ds-idrepo-0" 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG ] 01:14:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:14:48 DEBUG "result": [ 01:14:48 DEBUG ] 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG } 01:14:48 DEBUG }, 01:14:48 DEBUG { 01:14:48 DEBUG } 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG ] 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "metric": { 01:14:48 DEBUG ] 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG } 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:14:48 DEBUG } 01:14:48 DEBUG "0" 01:14:48 DEBUG "2301.6" 01:14:48 DEBUG } 01:14:48 DEBUG "0" 01:14:48 DEBUG }, 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG "value": [ 01:14:48 DEBUG } 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG 01:14:48 DEBUG ] 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG 01:14:48 DEBUG } 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:14:48 DEBUG } 01:14:48 DEBUG [...] 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG 01:14:48 DEBUG { 01:14:48 DEBUG 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG 01:14:48 DEBUG "data": { 01:14:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG }, 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG { 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "0" 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG ] 01:14:48 DEBUG }, 01:14:48 DEBUG } 01:14:48 DEBUG "value": [ 01:14:48 DEBUG ] 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 DEBUG 01:14:48 DEBUG } 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG 01:14:48 DEBUG } 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-idrepo-0" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "instance": "10.142.0.107:9100" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "instance": "10.142.0.43:9100" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0.00033333333333333365" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "2300.5" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-cts-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-idrepo-0" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053688" 01:14:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053688" 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "ds-idrepo-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "ds-cts-1" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "0.0001" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "2281.233333333333" 01:14:48 DEBUG "2295.5333333333333" 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "1856290816" 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 INFO [http_cmd]: http status code OK 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- status code --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG --- http response --- 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG ----- output ----- 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "status": "success", 01:14:48 DEBUG "data": { 01:14:48 DEBUG "data": { 01:14:48 DEBUG "data": { 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "resultType": "vector", 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "result": [ 01:14:48 DEBUG "result": [ 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "metric": { 01:14:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:14:48 DEBUG "instance": "10.142.0.107:9100" 01:14:48 DEBUG "instance": "10.142.0.107:9100" 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG [...] 01:14:48 DEBUG [...] 01:14:48 DEBUG [...] 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG 01:14:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:14:48 DEBUG "instance": "10.142.0.43:9100" 01:14:48 DEBUG "instance": "10.142.0.43:9100" 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG }, 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG "value": [ 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG 1713053688, 01:14:48 DEBUG "10.949203986572368" 01:14:48 DEBUG "12093.266666666665" 01:14:48 DEBUG "5310.933333333333" 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG ] 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:48 DEBUG } 01:14:51 INFO 01:14:51 INFO [loop_until]: kubectl --namespace=xlou top pods 01:14:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:14:51 INFO [loop_until]: OK (rc = 0) 01:14:51 DEBUG --- stdout --- 01:14:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 12342m 9018Mi am-7d597cf7d6-8vr8f 11403m 8399Mi am-7d597cf7d6-9wz9h 15184m 9095Mi ds-cts-0 11169m 7740Mi ds-cts-1 11660m 8040Mi ds-cts-2 12054m 8093Mi ds-idrepo-0 13288m 4277Mi ds-idrepo-1 2042m 4294Mi ds-idrepo-2 1828m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 970Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 9m 65Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 6507m 1679Mi 01:14:51 DEBUG --- stderr --- 01:14:51 DEBUG 01:14:52 INFO 01:14:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:14:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:14:52 INFO [loop_until]: OK (rc = 0) 01:14:52 DEBUG --- stdout --- 01:14:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 63m 0% 4797Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 11640m 38% 10252Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 12454m 41% 11084Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 17452m 58% 2977Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 2106m 7% 6306Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 11327m 37% 11205Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2899Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 12270m 41% 9836Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 1994m 6% 5467Mi 4% gke-xlou-cdm-ds-0f601751-j488 11759m 39% 9490Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2197Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 16440m 55% 10632Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 13237m 44% 5666Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 6668m 41% 5421Mi 9% 01:14:52 DEBUG --- stderr --- 01:14:52 DEBUG 01:14:56 WARNING File /tmp/lodemon_data-20240414_000936/node_disk_read_bytes_total.json does not exist. Skipping average value computation... 127.0.0.1 - - [14/Apr/2024 01:14:56] "GET /monitoring/average?start_time=20240413_234542&stop_time=20240414_001255 HTTP/1.1" 200 - 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053748" 01:15:48 ERROR [http_cmd]: ERROR 01:15:48 ERROR --- status code --- 01:15:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:15:48 ERROR --- http response --- 01:15:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:15:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "ds-idrepo-0" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0.024495131705830016" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0.013" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "337755" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "instance": "10.142.0.107:9100" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "instance": "10.142.0.43:9100" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0.0004999999999997821" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "ds-idrepo-0" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "instance": "10.142.0.107:9100" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "instance": "10.142.0.43:9100" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "ds-idrepo-0" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG { 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "instance": "10.142.0.107:9100" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:15:48 DEBUG 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG ----- output ----- 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG "instance": "10.142.0.43:9100" 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG { 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG }, 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG "value": [ 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG "data": { 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG "12605299712" 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG "result": [ 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG ] 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG { 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:15:48 DEBUG } 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG ] 01:15:48 DEBUG "data": { 01:15:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:15:48 DEBUG "data": { 01:15:48 DEBUG "data": { 01:15:48 DEBUG { 01:15:48 DEBUG "data": { 01:15:48 DEBUG { 01:15:48 DEBUG } 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG }, 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG } 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "data": { 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "data": { 01:15:48 DEBUG { 01:15:48 DEBUG 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "metric": { 01:15:48 DEBUG 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG 01:15:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:15:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:15:48 DEBUG { 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG { 01:15:48 DEBUG }, 01:15:48 DEBUG [...] 01:15:48 DEBUG }, 01:15:48 DEBUG }, 01:15:48 DEBUG "metric": { 01:15:48 DEBUG }, 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG }, 01:15:48 DEBUG 01:15:48 DEBUG }, 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG }, 01:15:48 DEBUG [...] 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG "38856756.35778031" 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG ] 01:15:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:15:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG 01:15:48 DEBUG }, 01:15:48 DEBUG } 01:15:48 DEBUG }, 01:15:48 DEBUG }, 01:15:48 DEBUG 01:15:48 DEBUG }, 01:15:48 DEBUG 01:15:48 DEBUG "value": [ 01:15:48 DEBUG ] 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "pod": "ds-idrepo-0" 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "pod": "ds-idrepo-0" 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG } 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG }, 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG }, 01:15:48 DEBUG "0" 01:15:48 DEBUG } 01:15:48 DEBUG "0" 01:15:48 DEBUG "0" 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "0" 01:15:48 DEBUG "value": [ 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG ] 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG "0" 01:15:48 DEBUG } 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "ds-cts-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "2279.2" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "ds-idrepo-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "ds-cts-1" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0.00030000000000000003" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "0" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG [...] 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 1713053748, 01:15:48 DEBUG "1253396480" 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG ] 01:15:48 DEBUG } 01:15:48 DEBUG } 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053748" 01:15:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053748" 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 INFO [http_cmd]: http status code OK 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG --- status code --- 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG --- http response --- 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (152 lines): 01:15:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG ----- output ----- 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "status": "success", 01:15:48 DEBUG "data": { 01:15:48 DEBUG "data": { 01:15:48 DEBUG "data": { 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "resultType": "vector", 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "result": [ 01:15:48 DEBUG "result": [ 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "metric": { 01:15:48 DEBUG "instance": "10.142.0.107:9100" 01:15:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:15:48 DEBUG "instance": "10.142.0.107:9100" 01:15:48 DEBUG }, 01:15:48 DEBUG }, 01:15:48 DEBUG }, 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "value": [ 01:15:48 DEBUG "value": [ 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:48 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG [...] 01:15:49 DEBUG [...] 01:15:49 DEBUG [...] 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG 01:15:49 DEBUG "instance": "10.142.0.43:9100" 01:15:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:15:49 DEBUG "instance": "10.142.0.43:9100" 01:15:49 DEBUG }, 01:15:49 DEBUG }, 01:15:49 DEBUG }, 01:15:49 DEBUG "value": [ 01:15:49 DEBUG "value": [ 01:15:49 DEBUG "value": [ 01:15:49 DEBUG 1713053748, 01:15:49 DEBUG 1713053748, 01:15:49 DEBUG 1713053748, 01:15:49 DEBUG "193597.96666666665" 01:15:49 DEBUG "0.0016768592565202024" 01:15:49 DEBUG "8139.766666666666" 01:15:49 DEBUG ] 01:15:49 DEBUG ] 01:15:49 DEBUG ] 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:49 DEBUG ] 01:15:49 DEBUG ] 01:15:49 DEBUG ] 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:49 DEBUG } 01:15:51 INFO 01:15:51 INFO [loop_until]: kubectl --namespace=xlou top pods 01:15:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:15:51 INFO [loop_until]: OK (rc = 0) 01:15:51 DEBUG --- stdout --- 01:15:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 2059m 9014Mi am-7d597cf7d6-8vr8f 2056m 8397Mi am-7d597cf7d6-9wz9h 2575m 9093Mi ds-cts-0 5306m 7749Mi ds-cts-1 5490m 7990Mi ds-cts-2 5345m 8106Mi ds-idrepo-0 1247m 4277Mi ds-idrepo-1 171m 4293Mi ds-idrepo-2 165m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 970Mi idm-b85f9d978-rl76f 4m 973Mi lodemon-64cbdcc7b4-vt4kk 11m 65Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 2m 1104Mi 01:15:51 DEBUG --- stderr --- 01:15:51 DEBUG 01:15:52 INFO 01:15:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:15:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:15:52 INFO [loop_until]: OK (rc = 0) 01:15:52 DEBUG --- stdout --- 01:15:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 69m 0% 4797Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5552m 18% 10241Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2194m 7% 11135Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 94m 0% 2989Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 216m 0% 6290Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 5242m 17% 11242Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 72m 0% 2917Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 2259m 7% 9869Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 221m 0% 5469Mi 4% gke-xlou-cdm-ds-0f601751-j488 5545m 18% 9486Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 64m 0% 2204Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 2766m 9% 10693Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 1312m 4% 5673Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 74m 0% 4854Mi 8% 01:15:52 DEBUG --- stderr --- 01:15:52 DEBUG 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:16:48 ERROR [http_cmd]: ERROR 01:16:48 ERROR --- status code --- 01:16:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:16:48 ERROR --- http response --- 01:16:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:16:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "1.0746165987530412" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "pod": "ds-idrepo-1" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "pod": "ds-idrepo-0" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "0" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053808" 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:48 DEBUG { 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG { 01:16:48 DEBUG "data": { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "result": [ 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG { 01:16:48 DEBUG "result": [ 01:16:48 DEBUG "metric": { 01:16:48 DEBUG { 01:16:48 DEBUG "pod": "ds-idrepo-1" 01:16:48 DEBUG "metric": { 01:16:48 DEBUG }, 01:16:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:48 DEBUG "value": [ 01:16:48 DEBUG }, 01:16:48 DEBUG 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:48 DEBUG 01:16:48 DEBUG "pod": "ds-idrepo-0" 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG 01:16:48 DEBUG }, 01:16:48 DEBUG { 01:16:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:48 DEBUG "value": [ 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG }, 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "data": { 01:16:48 DEBUG "value": [ 01:16:48 DEBUG "0" 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG ] 01:16:48 DEBUG "result": [ 01:16:48 DEBUG "0.013" 01:16:48 DEBUG } 01:16:48 DEBUG { 01:16:48 DEBUG ] 01:16:48 DEBUG ] 01:16:48 DEBUG "metric": { 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG }, 01:16:48 DEBUG } 01:16:48 DEBUG "value": [ 01:16:48 DEBUG } 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "200546" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "instance": "10.142.0.107:9100" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "instance": "10.142.0.43:9100" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "0" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "instance": "10.142.0.107:9100" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "instance": "10.142.0.43:9100" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "0.0004666666666667633" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "pod": "ds-idrepo-0" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "11702.477192298951" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "pod": "ds-idrepo-1" 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:48 DEBUG }, 01:16:48 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "pod": "ds-idrepo-0" 01:16:48 DEBUG 01:16:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:48 DEBUG }, 01:16:48 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "0" 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "0" 01:16:48 DEBUG ] 01:16:48 DEBUG "0" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 DEBUG --- status code --- 01:16:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:48 DEBUG --- http response --- 01:16:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:48 DEBUG ----- output ----- 01:16:48 DEBUG { 01:16:48 DEBUG "status": "success", 01:16:48 DEBUG "data": { 01:16:48 DEBUG "resultType": "vector", 01:16:48 DEBUG "result": [ 01:16:48 DEBUG { 01:16:48 DEBUG "metric": { 01:16:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG [...] 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG 01:16:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:48 DEBUG }, 01:16:48 DEBUG "value": [ 01:16:48 DEBUG 1713053808, 01:16:48 DEBUG "0" 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG ] 01:16:48 DEBUG } 01:16:48 DEBUG } 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 INFO [http_cmd]: http status code OK 01:16:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053808" 01:16:48 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "instance": "10.142.0.107:9100" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "instance": "10.142.0.43:9100" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053808, 01:16:49 DEBUG "12605965312" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "ds-idrepo-1" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "ds-cts-1" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053808, 01:16:49 DEBUG "0" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053808, 01:16:49 DEBUG "0" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "ds-idrepo-0" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053808, 01:16:49 DEBUG "0" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "ds-idrepo-1" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "ds-idrepo-0" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053808, 01:16:49 DEBUG "0" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "ds-idrepo-1" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "ds-idrepo-0" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053808, 01:16:49 DEBUG "0" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053809" 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053809" 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053809" 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:16:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG }, 01:16:49 DEBUG { 01:16:49 DEBUG "value": [ 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG 01:16:49 DEBUG "data": { 01:16:49 DEBUG 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG 01:16:49 DEBUG "result": [ 01:16:49 DEBUG [...] 01:16:49 DEBUG { 01:16:49 DEBUG 01:16:49 DEBUG "metric": { 01:16:49 DEBUG 01:16:49 DEBUG "pod": "ds-idrepo-1" 01:16:49 DEBUG 01:16:49 DEBUG }, 01:16:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:49 DEBUG "value": [ 01:16:49 DEBUG }, 01:16:49 DEBUG 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG 01:16:49 DEBUG "2283.7" 01:16:49 DEBUG [...] 01:16:49 DEBUG ] 01:16:49 DEBUG 01:16:49 DEBUG } 01:16:49 DEBUG 01:16:49 DEBUG ] 01:16:49 DEBUG 01:16:49 DEBUG } 01:16:49 DEBUG "pod": "ds-cts-1" 01:16:49 DEBUG } 01:16:49 DEBUG }, 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053809" 01:16:49 DEBUG "value": [ 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG "0.00006666666666666667" 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG ] 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG "0" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG "1937231872" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053809" 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053809" 01:16:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053809" 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "instance": "10.142.0.107:9100" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "instance": "10.142.0.43:9100" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG "12341.733333333335" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "pod": "ds-idrepo-0" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG "1.7677006590694153" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:49 INFO [http_cmd]: http status code OK 01:16:49 DEBUG --- status code --- 01:16:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:16:49 DEBUG --- http response --- 01:16:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:16:49 DEBUG ----- output ----- 01:16:49 DEBUG { 01:16:49 DEBUG "status": "success", 01:16:49 DEBUG "data": { 01:16:49 DEBUG "resultType": "vector", 01:16:49 DEBUG "result": [ 01:16:49 DEBUG { 01:16:49 DEBUG "metric": { 01:16:49 DEBUG "instance": "10.142.0.107:9100" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG [...] 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG 01:16:49 DEBUG "instance": "10.142.0.43:9100" 01:16:49 DEBUG }, 01:16:49 DEBUG "value": [ 01:16:49 DEBUG 1713053809, 01:16:49 DEBUG "7850.699999999999" 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG ] 01:16:49 DEBUG } 01:16:49 DEBUG } 01:16:51 INFO 01:16:51 INFO [loop_until]: kubectl --namespace=xlou top pods 01:16:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:16:51 INFO [loop_until]: OK (rc = 0) 01:16:51 DEBUG --- stdout --- 01:16:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 2097m 8910Mi am-7d597cf7d6-8vr8f 1925m 8355Mi am-7d597cf7d6-9wz9h 2491m 8998Mi ds-cts-0 5250m 7768Mi ds-cts-1 5434m 8013Mi ds-cts-2 5450m 8072Mi ds-idrepo-0 1748m 4286Mi ds-idrepo-1 491m 4294Mi ds-idrepo-2 430m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 5m 973Mi idm-b85f9d978-rl76f 4m 977Mi lodemon-64cbdcc7b4-vt4kk 8m 65Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 1086m 1592Mi 01:16:51 DEBUG --- stderr --- 01:16:51 DEBUG 01:16:52 INFO 01:16:52 INFO [loop_until]: kubectl --namespace=xlou top node 01:16:52 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:16:52 INFO [loop_until]: OK (rc = 0) 01:16:52 DEBUG --- stdout --- 01:16:52 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 65m 0% 4805Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5291m 17% 10258Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2158m 7% 11009Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 91m 0% 3001Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 480m 1% 6271Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 5171m 17% 11253Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 67m 0% 2918Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 2171m 7% 9821Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 500m 1% 5420Mi 4% gke-xlou-cdm-ds-0f601751-j488 5562m 18% 9478Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 61m 0% 2202Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 2741m 9% 10576Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 1726m 5% 5660Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 1077m 6% 5382Mi 9% 01:16:52 DEBUG --- stderr --- 01:16:52 DEBUG 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053868" 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053868" 01:17:48 ERROR [http_cmd]: ERROR 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 ERROR --- status code --- 01:17:48 DEBUG --- status code --- 01:17:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 ERROR --- http response --- 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:17:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:17:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053868" 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053868" 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 DEBUG --- status code --- 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:48 DEBUG ----- output ----- 01:17:48 DEBUG { 01:17:48 DEBUG "status": "success", 01:17:48 DEBUG "data": { 01:17:48 DEBUG "resultType": "vector", 01:17:48 DEBUG "result": [ 01:17:48 DEBUG { 01:17:48 DEBUG "metric": { 01:17:48 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG [...] 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 1713053868, 01:17:48 DEBUG "1.0314249357521952" 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG } 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053868" 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053868" 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053868" 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 DEBUG --- status code --- 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:17:48 DEBUG ----- output ----- 01:17:48 DEBUG { 01:17:48 DEBUG "status": "success", 01:17:48 DEBUG "data": { 01:17:48 DEBUG "resultType": "vector", 01:17:48 DEBUG "result": [ 01:17:48 DEBUG { 01:17:48 DEBUG "metric": { 01:17:48 DEBUG "pod": "ds-idrepo-1" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG [...] 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG "pod": "ds-idrepo-0" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 1713053868, 01:17:48 DEBUG "0" 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG } 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 DEBUG --- status code --- 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:48 DEBUG ----- output ----- 01:17:48 DEBUG { 01:17:48 DEBUG "status": "success", 01:17:48 DEBUG "data": { 01:17:48 DEBUG "resultType": "vector", 01:17:48 DEBUG "result": [ 01:17:48 DEBUG { 01:17:48 DEBUG "metric": { 01:17:48 DEBUG "pod": "ds-idrepo-1" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG [...] 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG "pod": "ds-idrepo-0" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 1713053868, 01:17:48 DEBUG "0" 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG } 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 DEBUG --- status code --- 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:48 DEBUG ----- output ----- 01:17:48 DEBUG { 01:17:48 DEBUG "status": "success", 01:17:48 DEBUG "data": { 01:17:48 DEBUG "resultType": "vector", 01:17:48 DEBUG "result": [ 01:17:48 DEBUG { 01:17:48 DEBUG "metric": { 01:17:48 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG [...] 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 1713053868, 01:17:48 DEBUG "63382" 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG } 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 DEBUG --- status code --- 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:48 DEBUG ----- output ----- 01:17:48 DEBUG { 01:17:48 DEBUG "status": "success", 01:17:48 DEBUG "data": { 01:17:48 DEBUG "resultType": "vector", 01:17:48 DEBUG "result": [ 01:17:48 DEBUG { 01:17:48 DEBUG "metric": { 01:17:48 DEBUG "instance": "10.142.0.107:9100" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG [...] 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG "instance": "10.142.0.43:9100" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 1713053868, 01:17:48 DEBUG "0.0005666666666667671" 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG } 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053868" 01:17:48 INFO [http_cmd]: http status code OK 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053868" 01:17:48 DEBUG --- status code --- 01:17:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053868" 01:17:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:48 DEBUG --- http response --- 01:17:48 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:48 DEBUG ----- output ----- 01:17:48 DEBUG { 01:17:48 DEBUG "status": "success", 01:17:48 DEBUG "data": { 01:17:48 DEBUG "resultType": "vector", 01:17:48 DEBUG "result": [ 01:17:48 DEBUG { 01:17:48 DEBUG "metric": { 01:17:48 DEBUG "pod": "ds-idrepo-1" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG [...] 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG 01:17:48 DEBUG "pod": "ds-idrepo-0" 01:17:48 DEBUG }, 01:17:48 DEBUG "value": [ 01:17:48 DEBUG 1713053868, 01:17:48 DEBUG "0" 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG ] 01:17:48 DEBUG } 01:17:48 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053868, 01:17:49 DEBUG "0.013" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "instance": "10.142.0.107:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "instance": "10.142.0.43:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053868, 01:17:49 DEBUG "0.00033333333333333365" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "ds-idrepo-1" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "ds-idrepo-0" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "data": { 01:17:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG }, 01:17:49 DEBUG "result": [ 01:17:49 DEBUG "value": [ 01:17:49 DEBUG { 01:17:49 DEBUG 01:17:49 DEBUG "metric": { 01:17:49 DEBUG 01:17:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:17:49 DEBUG 01:17:49 DEBUG }, 01:17:49 DEBUG [...] 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:49 DEBUG [...] 01:17:49 DEBUG }, 01:17:49 DEBUG 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG 01:17:49 DEBUG "0" 01:17:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:17:49 DEBUG ] 01:17:49 DEBUG }, 01:17:49 DEBUG } 01:17:49 DEBUG "value": [ 01:17:49 DEBUG ] 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG } 01:17:49 DEBUG "11939953.16212889" 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "964.3333333333334" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:49 DEBUG ] 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG } 01:17:49 DEBUG { 01:17:49 DEBUG } 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "instance": "10.142.0.107:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "instance": "10.142.0.43:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "12606616576" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "ds-idrepo-1" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "ds-cts-1" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG { 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "ds-idrepo-1" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "ds-idrepo-0" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "instance": "10.142.0.107:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "instance": "10.142.0.43:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "10169.566666666668" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "ds-idrepo-1" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "ds-cts-1" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "0" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "2544316416" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053869" 01:17:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053869" 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG --- http response --- 01:17:49 INFO [http_cmd]: http status code OK 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:17:49 DEBUG --- status code --- 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:17:49 DEBUG { 01:17:49 DEBUG --- http response --- 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "1.0043059279681938" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:17:49 DEBUG ----- output ----- 01:17:49 DEBUG { 01:17:49 DEBUG "status": "success", 01:17:49 DEBUG "data": { 01:17:49 DEBUG "resultType": "vector", 01:17:49 DEBUG "result": [ 01:17:49 DEBUG { 01:17:49 DEBUG "metric": { 01:17:49 DEBUG "instance": "10.142.0.107:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG [...] 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG 01:17:49 DEBUG "instance": "10.142.0.43:9100" 01:17:49 DEBUG }, 01:17:49 DEBUG "value": [ 01:17:49 DEBUG 1713053869, 01:17:49 DEBUG "11989.699999999999" 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG ] 01:17:49 DEBUG } 01:17:49 DEBUG } 01:17:51 INFO 01:17:51 INFO [loop_until]: kubectl --namespace=xlou top pods 01:17:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:17:51 INFO [loop_until]: OK (rc = 0) 01:17:51 DEBUG --- stdout --- 01:17:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 2138m 8907Mi am-7d597cf7d6-8vr8f 1953m 8354Mi am-7d597cf7d6-9wz9h 2508m 8998Mi ds-cts-0 4895m 7761Mi ds-cts-1 5138m 7945Mi ds-cts-2 5054m 8053Mi ds-idrepo-0 1246m 4286Mi ds-idrepo-1 172m 4294Mi ds-idrepo-2 161m 4114Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 973Mi idm-b85f9d978-rl76f 4m 977Mi lodemon-64cbdcc7b4-vt4kk 6m 65Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 987m 2332Mi 01:17:51 DEBUG --- stderr --- 01:17:51 DEBUG 01:17:53 INFO 01:17:53 INFO [loop_until]: kubectl --namespace=xlou top node 01:17:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:17:53 INFO [loop_until]: OK (rc = 0) 01:17:53 DEBUG --- stdout --- 01:17:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 62m 0% 4809Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 5168m 17% 10241Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 2185m 7% 11009Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 2996Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 217m 0% 6297Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 4984m 16% 11259Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2913Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 2179m 7% 9821Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 220m 0% 5458Mi 4% gke-xlou-cdm-ds-0f601751-j488 5265m 17% 9473Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 62m 0% 2202Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 2793m 9% 10583Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 1330m 4% 5677Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 1077m 6% 6072Mi 10% 01:17:53 DEBUG --- stderr --- 01:17:53 DEBUG 01:18:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053928" 01:18:48 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053928" 01:18:48 ERROR [http_cmd]: ERROR 01:18:48 INFO [http_cmd]: http status code OK 01:18:48 ERROR --- status code --- 01:18:48 DEBUG --- status code --- 01:18:48 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:18:48 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:48 ERROR --- http response --- 01:18:48 DEBUG --- http response --- 01:18:48 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:18:48 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:18:48 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-idrepo-0" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 DEBUG --- status code --- 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "instance": "10.142.0.107:9100" 01:18:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:18:49 DEBUG }, 01:18:49 DEBUG }, 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG "value": [ 01:18:49 DEBUG "value": [ 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG { 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG 01:18:49 DEBUG "instance": "10.142.0.43:9100" 01:18:49 DEBUG "data": { 01:18:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:18:49 DEBUG }, 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG "result": [ 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG { 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0.0006333333333332783" 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "0.7000726130886044" 01:18:49 DEBUG ] 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG }, 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG "value": [ 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 DEBUG 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG } 01:18:49 DEBUG 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [...] 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:18:49 DEBUG 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG 01:18:49 DEBUG { 01:18:49 DEBUG 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG "data": { 01:18:49 DEBUG }, 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "value": [ 01:18:49 DEBUG "result": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG { 01:18:49 DEBUG "0.013" 01:18:49 DEBUG "metric": { 01:18:49 DEBUG ] 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG }, 01:18:49 DEBUG ] 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG "value": [ 01:18:49 DEBUG } 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG 01:18:49 DEBUG } 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-idrepo-0" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-idrepo-0" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "1" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "instance": "10.142.0.107:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "instance": "10.142.0.43:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0.0009999999999999935" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-idrepo-0" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-idrepo-0" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "4202.363263284512" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "8871913.394183826" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-cts-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (125 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0.06666666666666667" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "instance": "10.142.0.107:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "instance": "10.142.0.43:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "12607370240" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053929" 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "instance": "10.142.0.107:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "instance": "10.142.0.43:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "12045.033333333331" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "instance": "10.142.0.107:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "instance": "10.142.0.43:9100" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "4768.633333333333" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "ds-idrepo-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "ds-cts-1" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "2908192768" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053929" 01:18:49 INFO [http_cmd]: http status code OK 01:18:49 DEBUG --- status code --- 01:18:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:18:49 DEBUG --- http response --- 01:18:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:18:49 DEBUG ----- output ----- 01:18:49 DEBUG { 01:18:49 DEBUG "status": "success", 01:18:49 DEBUG "data": { 01:18:49 DEBUG "resultType": "vector", 01:18:49 DEBUG "result": [ 01:18:49 DEBUG { 01:18:49 DEBUG "metric": { 01:18:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG [...] 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG 01:18:49 DEBUG "pod": "overseer-0-5584f6c4db-bcn5q" 01:18:49 DEBUG }, 01:18:49 DEBUG "value": [ 01:18:49 DEBUG 1713053929, 01:18:49 DEBUG "0.5228190323356067" 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG ] 01:18:49 DEBUG } 01:18:49 DEBUG } 01:18:51 INFO 01:18:51 INFO [loop_until]: kubectl --namespace=xlou top pods 01:18:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:18:51 INFO [loop_until]: OK (rc = 0) 01:18:51 DEBUG --- stdout --- 01:18:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 7m 8903Mi am-7d597cf7d6-8vr8f 7m 8353Mi am-7d597cf7d6-9wz9h 7m 8992Mi ds-cts-0 17m 7759Mi ds-cts-1 16m 7943Mi ds-cts-2 16m 8052Mi ds-idrepo-0 14m 4284Mi ds-idrepo-1 17m 4294Mi ds-idrepo-2 10m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 16m 974Mi idm-b85f9d978-rl76f 4m 977Mi lodemon-64cbdcc7b4-vt4kk 2m 65Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 419m 2549Mi 01:18:51 DEBUG --- stderr --- 01:18:51 DEBUG 01:18:53 INFO 01:18:53 INFO [loop_until]: kubectl --namespace=xlou top node 01:18:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:18:53 INFO [loop_until]: OK (rc = 0) 01:18:53 DEBUG --- stdout --- 01:18:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 64m 0% 4813Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 74m 0% 10226Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 55m 0% 11013Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 88m 0% 2988Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 65m 0% 6298Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 167m 0% 11244Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 77m 0% 2911Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 62m 0% 9819Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 66m 0% 5448Mi 4% gke-xlou-cdm-ds-0f601751-j488 69m 0% 9461Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 67m 0% 2205Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 90m 0% 10565Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 70m 0% 5674Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 691m 4% 6289Mi 10% 01:18:53 DEBUG --- stderr --- 01:18:53 DEBUG 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_read_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%29%29by%28instance%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_entry_count%7Bbackend%3D~%27amCts%27%2Cpod%3D~%27ctsstore.%2A%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27cfgStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 ERROR [http_cmd]: ERROR 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG { "status": "success", "data": { "resultType": "vector", "result": [] } } 01:19:49 ERROR --- status code --- 01:19:49 DEBUG --- status code --- 01:19:49 ERROR http status code is 400 (expected [200, 401, 403, 404]) 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 ERROR --- http response --- 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG { "status": "error", "errorType": "bad_data", "error": "invalid parameter \"query\": 1:10: parse error: expected type range vector in call to function \"rate\", got instant vector" } 01:19:49 DEBUG ----- output ----- 01:19:49 ERROR Query: sum(rate(node_disk_read_bytes_total{job='node-exporter',device=~'nvme.+|rbd.+|sd.+|vd.+|xvd.+|dasd.+'}))by(instance) failed with: [http_cmd] request returns wrong status code 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "ds-idrepo-0" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_disk_io_time_seconds_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_cpu_usage_seconds_total%3Asum_irate%29by%28node%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_search_seconds_total%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amCts%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27idmRepo%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_authentication_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28am_cts_reaper_cache_size%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_cpu_seconds_total%7Bmode%3D%27iowait%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG { 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:19:49 DEBUG "data": { 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG { 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 DEBUG "status": "success", 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG { 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG "data": { 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG "metric": { 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG "data": { 01:19:49 DEBUG "data": { 01:19:49 DEBUG { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG "instance": "10.142.0.107:9100" 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG }, 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG "result": [ 01:19:49 DEBUG "result": [ 01:19:49 DEBUG "data": { 01:19:49 DEBUG { 01:19:49 DEBUG "value": [ 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG { 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "metric": { 01:19:49 DEBUG 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG "metric": { 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "result": [ 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG 01:19:49 DEBUG { 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG { 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG { 01:19:49 DEBUG }, 01:19:49 DEBUG 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG }, 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG }, 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "value": [ 01:19:49 DEBUG [...] 01:19:49 DEBUG "data": { 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "data": { 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG 01:19:49 DEBUG "resultType": "vector", 01:19:49 [35mDEBUG 01:19:49 DEBUG }, 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "result": [ 01:19:49 DEBUG 01:19:49 DEBUG "result": [ 01:19:49 DEBUG 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG { 01:19:49 DEBUG 01:19:49 DEBUG { 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG "instance": "10.142.0.43:9100" 01:19:49 DEBUG "metric": { 01:19:49 DEBUG [...] 01:19:49 DEBUG "metric": { 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG }, 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "value": [ 01:19:49 DEBUG }, 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "value": [ 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "ds-idrepo-0" 01:19:49 DEBUG "0.000500000000000019" 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG }, 01:19:49 DEBUG ] 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG 01:19:49 DEBUG }, 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "value": [ 01:19:49 DEBUG } 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG 01:19:49 DEBUG "value": [ 01:19:49 DEBUG }, 01:19:49 DEBUG [...] 01:19:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG ] 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG [...] 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG }, 01:19:49 DEBUG "0" 01:19:49 DEBUG } 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG 01:19:49 DEBUG "0" 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG 01:19:49 DEBUG "value": [ 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 DEBUG ] 01:19:49 DEBUG "0.013" 01:19:49 DEBUG 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG "1.0327322450752938" 01:19:49 DEBUG ] 01:19:49 DEBUG "pod": "ds-idrepo-0" 01:19:49 DEBUG { 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG }, 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG }, 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG "value": [ 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "data": { 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG ] 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG } 01:19:49 DEBUG "0" 01:19:49 DEBUG } 01:19:49 DEBUG "0" 01:19:49 DEBUG "result": [ 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG { 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG "metric": { 01:19:49 DEBUG ] 01:19:49 DEBUG ] 01:19:49 DEBUG "instance": "10.142.0.107:9100" 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG }, 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "instance": "10.142.0.43:9100" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0.00033333333333333365" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_replayed_updates_conflicts_resolved%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "ds-cts-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_ttl_entries_deleted_count%7Bnamespace%3D%27xlou%27%2Cbackend%3D~%27amCts%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_backend_db_cache_misses_internal_nodes%7Bbackend%3D%27amIdentityStore%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_writes_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (62 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "ds-idrepo-0" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG "result": [ 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG { 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG "metric": { 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "ds-idrepo-0" 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:19:49 DEBUG }, 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG "value": [ 01:19:49 DEBUG { 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "0" 01:19:49 DEBUG "data": { 01:19:49 DEBUG ] 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG } 01:19:49 DEBUG "result": [ 01:19:49 DEBUG ] 01:19:49 DEBUG { 01:19:49 DEBUG } 01:19:49 DEBUG "metric": { 01:19:49 DEBUG } 01:19:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "555.3936905495108" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_task_count%7Btoken_type%3D%27session%27%2Cnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_memory_working_set_bytes%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_fs_reads_bytes_total%7Bnamespace%3D%27xlou%27%2Cjob%3D%27kubelet%27%2Cmetrics_path%3D%27/metrics/cadvisor%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "21438.196455211113" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_oauth2_grant_count%7Bnamespace%3D%27xlou%27%2Cgrant_type%3D~%27authorization-code%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28irate%28am_session_count%7Bsession_type%3D~%27authentication-.%2A%27%2Coperation%3D%27create%27%2Cnamespace%3D%27xlou%27%2Coutcome%3D%27success%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28avg_over_time%28node_disk_written_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%3D~%27nvme.%2B%7Crbd.%2B%7Csd.%2B%7Cvd.%2B%7Cxvd.%2B%7Cdasd.%2B%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28am_cts_reaper_deletion_count%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_transmit_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28node_network_receive_bytes_total%7Bjob%3D%27node-exporter%27%2Cdevice%21%3D%27lo%27%7D%5B60s%5D%29%29by%28instance%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28container_cpu_usage_seconds_total%7Bnamespace%3D%27xlou%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (35 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG }, 01:19:49 DEBUG }, 01:19:49 DEBUG { 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "data": { 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "result": [ 01:19:49 DEBUG [...] 01:19:49 DEBUG [...] 01:19:49 DEBUG { 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "metric": { 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "instance": "10.142.0.107:9100" 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG }, 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG "pod": "am-7d597cf7d6-9wz9h" 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG "value": [ 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG }, 01:19:49 DEBUG }, 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "value": [ 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 DEBUG "0" 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 DEBUG "0" 01:19:49 DEBUG [...] 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG ] 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG ] 01:19:49 DEBUG 01:19:49 DEBUG { 01:19:49 DEBUG } 01:19:49 DEBUG { 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG ] 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG ] 01:19:49 DEBUG 01:19:49 DEBUG "data": { 01:19:49 DEBUG } 01:19:49 DEBUG "data": { 01:19:49 DEBUG } 01:19:49 DEBUG "instance": "10.142.0.43:9100" 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG } 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG } 01:19:49 DEBUG }, 01:19:49 DEBUG "result": [ 01:19:49 DEBUG "result": [ 01:19:49 DEBUG "value": [ 01:19:49 DEBUG { 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG { 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "metric": { 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "12608064512" 01:19:49 DEBUG "instance": "10.142.0.107:9100" 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG "instance": "10.142.0.107:9100" 01:19:49 DEBUG ] 01:19:49 DEBUG }, 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG }, 01:19:49 DEBUG } 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "value": [ 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (143 lines): 01:19:49 DEBUG ] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG { 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG [...] 01:19:49 DEBUG [...] 01:19:49 DEBUG "data": { 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "result": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG { 01:19:49 DEBUG "instance": "10.142.0.43:9100" 01:19:49 DEBUG "instance": "10.142.0.43:9100" 01:19:49 DEBUG "metric": { 01:19:49 DEBUG }, 01:19:49 DEBUG }, 01:19:49 DEBUG "pod": "admin-ui-c4c7bf584-mbzlq" 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "value": [ 01:19:49 DEBUG }, 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG "12796.966666666665" 01:19:49 DEBUG "3772.333333333333" 01:19:49 DEBUG 01:19:49 DEBUG ] 01:19:49 DEBUG ] 01:19:49 DEBUG 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG ] 01:19:49 DEBUG ] 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 DEBUG 01:19:49 DEBUG "pod": "am-7d597cf7d6-8ctft" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0.0026526014867076513" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28node_namespace_pod_container%3Acontainer_memory_working_set_bytes%29by%28node%29&time=1713053989" 01:19:49 INFO [http_cmd]: curl --insecure -L --request GET "http://prometheus-operator-kube-p-prometheus.monitoring.svc.cluster.local:9090/api/v1/query?query=sum%28rate%28ds_replication_replica_remote_replicas_receive_delay_seconds+%7Bnamespace%3D%27xlou%27%2Cdomain_name%3D%27ou%3Dtokens%27%7D%5B60s%5D%29%29by%28pod%29&time=1713053989" 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (134 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "node": "gke-xlou-cdm-ds-0f601751-mc5d" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "node": "gke-xlou-cdm-frontend-a8771548-i82n" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "3570245632" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:49 INFO [http_cmd]: http status code OK 01:19:49 DEBUG --- status code --- 01:19:49 DEBUG http status code is 200 (expected [200, 401, 403, 404]) 01:19:49 DEBUG --- http response --- 01:19:49 DEBUG [print_head_tail]: Print head (10 lines) and tail (10 lines) of input string (53 lines): 01:19:49 DEBUG ----- output ----- 01:19:49 DEBUG { 01:19:49 DEBUG "status": "success", 01:19:49 DEBUG "data": { 01:19:49 DEBUG "resultType": "vector", 01:19:49 DEBUG "result": [ 01:19:49 DEBUG { 01:19:49 DEBUG "metric": { 01:19:49 DEBUG "pod": "ds-idrepo-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG [...] 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG 01:19:49 DEBUG "pod": "ds-cts-1" 01:19:49 DEBUG }, 01:19:49 DEBUG "value": [ 01:19:49 DEBUG 1713053989, 01:19:49 DEBUG "0" 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG ] 01:19:49 DEBUG } 01:19:49 DEBUG } 01:19:51 INFO 01:19:51 INFO [loop_until]: kubectl --namespace=xlou top pods 01:19:51 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:19:51 INFO [loop_until]: OK (rc = 0) 01:19:51 DEBUG --- stdout --- 01:19:51 DEBUG NAME CPU(cores) MEMORY(bytes) admin-ui-c4c7bf584-mbzlq 1m 8Mi am-7d597cf7d6-8ctft 6m 8884Mi am-7d597cf7d6-8vr8f 8m 8341Mi am-7d597cf7d6-9wz9h 5m 8972Mi ds-cts-0 17m 7761Mi ds-cts-1 15m 7943Mi ds-cts-2 16m 8052Mi ds-idrepo-0 19m 4285Mi ds-idrepo-1 10m 4294Mi ds-idrepo-2 16m 4113Mi end-user-ui-58878d7668-zg4h2 1m 5Mi idm-b85f9d978-j5pbm 4m 974Mi idm-b85f9d978-rl76f 6m 977Mi lodemon-64cbdcc7b4-vt4kk 2m 65Mi login-ui-5fdf49fc66-2wnx7 1m 4Mi overseer-0-5584f6c4db-bcn5q 1025m 3309Mi 01:19:51 DEBUG --- stderr --- 01:19:51 DEBUG 01:19:53 INFO 01:19:53 INFO [loop_until]: kubectl --namespace=xlou top node 01:19:53 INFO [loop_until]: (max_time=180, interval=5, expected_rc=[0] 01:19:53 INFO [loop_until]: OK (rc = 0) 01:19:53 DEBUG --- stdout --- 01:19:53 DEBUG NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% gke-xlou-cdm-default-pool-1b0769c8-7vmj 67m 0% 4810Mi 4% gke-xlou-cdm-default-pool-1b0769c8-mpfw 76m 0% 10203Mi 9% gke-xlou-cdm-default-pool-1b0769c8-nhlf 54m 0% 10992Mi 9% gke-xlou-cdm-default-pool-1b0769c8-qkns 89m 0% 2995Mi 2% gke-xlou-cdm-default-pool-1b0769c8-w15q 67m 0% 6296Mi 5% gke-xlou-cdm-default-pool-1b0769c8-wpwl 147m 0% 11234Mi 10% gke-xlou-cdm-default-pool-1b0769c8-xnmp 65m 0% 2916Mi 2% gke-xlou-cdm-ds-0f601751-cf7b 62m 0% 9806Mi 8% gke-xlou-cdm-ds-0f601751-hmm7 69m 0% 5457Mi 4% gke-xlou-cdm-ds-0f601751-j488 73m 0% 9452Mi 8% gke-xlou-cdm-ds-0f601751-mc5d 71m 0% 2207Mi 1% gke-xlou-cdm-ds-0f601751-nj6q 83m 0% 10540Mi 9% gke-xlou-cdm-ds-0f601751-nmxm 74m 0% 5672Mi 5% gke-xlou-cdm-frontend-a8771548-i82n 1041m 6% 7134Mi 12% 01:19:53 DEBUG --- stderr --- 01:19:53 DEBUG 01:20:22 INFO Finished: True 01:20:22 INFO Waiting for threads to register finish flag 01:20:53 INFO Done. Have a nice day! :) 127.0.0.1 - - [14/Apr/2024 01:20:53] "GET /monitoring/stop HTTP/1.1" 200 - 01:20:55 WARNING File /tmp/lodemon_data-20240414_000936/node_disk_read_bytes_total.json does not exist. Skipping graph generation... WARNING:bokeh.core.validation.check:W-1000 (MISSING_RENDERERS): Plot has no renderers: figure(id='p6471', ...) 127.0.0.1 - - [14/Apr/2024 01:20:59] "GET /monitoring/process HTTP/1.1" 200 -