Skip to content

Commit fb68895

Browse files
alosadagrandealosadagrande
committed
Adds script to scrap prometheus data and ingest as a friendy splunk log
Signed-off-by: Alberto Losada <alosadag@redhat.com>
1 parent 35b38ac commit fb68895

File tree

1 file changed

+45
-0
lines changed

1 file changed

+45
-0
lines changed

‎splunk/resource-usage-splunk.sh

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
3+
SNO_HOSTNAME="${1:-zt-sno3}"
4+
NON_RESERVED_CORES="2-31,34-63"
5+
6+
# NODE CPU AVG
7+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=100 * avg(1 - rate(node_cpu_seconds_total{mode="idle"}[30m])) by (instance)' | jq -r '.data.result[] | [.value[0], .value[1], .metric.instance] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_avg_node_noidle_cpu_percentage.txt
8+
9+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=100 * max(1 - rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)' | jq -r '.data.result[] | [.value[0], .value[1], .metric.instance] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_max_node_noidle_cpu_percentage.txt
10+
11+
# NODE MEMORY
12+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=100 * (1 - (sum(avg_over_time(node_memory_MemAvailable_bytes{job="node-exporter"}[30m]))/sum(avg_over_time(node_memory_MemTotal_bytes{job="node-exporter"}[30m]))))' | jq -r '.data.result[] | [.value[0], .value[1]] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_node_percentage_free_memory.txt
13+
14+
# CONTAINER CPU USAGE SLICES
15+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc((rate(container_cpu_usage_seconds_total{id=~"/system.slice/.+"}[30m])))' | jq -r '.data.result[] | [.value[0], .value[1], .metric.cpu, .metric.service, .metric.id] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_system-slice-cpu.txt
16+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc((rate(container_cpu_usage_seconds_total{id=~"/ovs.slice/.+"}[30m])))' | jq -r '.data.result[] | [.value[0], .value[1], .metric.cpu, .metric.service, .metric.id] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_ovs-slice-cpu.txt
17+
18+
# CONTAINER CPU USAGE
19+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(avg_over_time(pod:container_cpu_usage:sum[30m]))' | jq -r '.data.result[] | [.value[0], .value[1], .metric.namespace, .metric.pod] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_pod-cpu.txt
20+
21+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(rate(container_cpu_cfs_throttled_periods_total[30m]))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.service, .metric.endpoint, .metric.id, .metric.namespace, .metric.pod, .metric.container] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_container_cpu_cfs_throttled_periods_total.txt
22+
23+
## NODE RESERVED CPU USAGE Replace 0|1|32|33 with the reserved cpus
24+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=rate(node_cpu_seconds_total{cpu=~"0|1|32|33"}[30m])' | jq -r '.data.result[] | [.value[0], .value[1], .metric.cpu, .metric.endpoint, .metric.mode, .metric.service, .metric.namespace, .metric.pod, .metric.container] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_reserved-cpu-usage.txt
25+
26+
# Container Memory
27+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(sum(container_memory_working_set_bytes{cluster="",container!=""}) by (pod))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.pod] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_memory-working-set-bytes.txt
28+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(sum(container_memory_rss{cluster="",container!=""}) by (pod))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.pod] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_memory-rss-bytes.txt
29+
30+
# NODE Networking
31+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(rate(node_network_receive_bytes_total[30m]))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.device, .metric.endpoint, .metric.service, .metric.namespace, .metric.pod, .metric.container] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_node-network-receive-bytes-total.txt
32+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(rate(node_network_transmit_bytes_total[30m]))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.device, .metric.endpoint, .metric.service, .metric.namespace, .metric.pod, .metric.container] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_node-network-transmit-bytes-total.txt
33+
34+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sum(irate(node_network_receive_bytes_total[30m])*8/(1024*1024)) by (instance)' |jq -r '.data.result[] | [.value[0], .value[1], .metric.instance] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_node-network-receive-mbps-total.txt
35+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sum(irate(node_network_transmit_bytes_total[30m])*8/(1024*1024)) by (instance)' |jq -r '.data.result[] | [.value[0], .value[1], .metric.instance] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_node-network-transmit-mbps-total.txt
36+
37+
38+
# CONTAINER Networking
39+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(rate(container_network_transmit_bytes_total{interface="eth0"}[30m]))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.name, .metric.id, .metric.interface, .metric.service, .metric.namespace, .metric.pod, .metric.container] | @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_container-network-transmit-bytes-total.txt
40+
oc --kubeconfig=/root/${SNO_HOSTNAME}/kubeconfig rsh -n openshift-monitoring prometheus-k8s-0 curl -ks 'http://localhost:9090/api/v1/query' --data-urlencode 'query=sort_desc(rate(container_network_receive_bytes_total{interface="eth0"}[30m]))' |jq -r '.data.result[] | [.value[0], .value[1], .metric.name, .metric.id, .metric.interface, .metric.service, .metric.namespace, .metric.pod, .metric.container]| @tsv' | sed 's/\t/ /g' >> ${SNO_HOSTNAME}_container-network-receive-bytes-total.txt
41+
42+
# On the SNO
43+
#Collect after the above
44+
ssh core@${SNO_HOSTNAME} sudo cat /proc/meminfo > ${SNO_HOSTNAME}_proc_meminfo.txt
45+
#taskset -c ${NON_RESERVED_CORES} top -b -1 -d 5 -n 360 -i > /home/core/top.out

0 commit comments

Comments
 (0)