Mercurial > repos > other > usr-local-bin

annotate load-record @ 29:c2584db4a650

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Make load recording more flexible and efficient * Works with variable number of CPUs * Use Python `enumerate()` function where possible * Use long-running NVidia function
author IBBoard <dev@ibboard.co.uk>
date Wed, 30 Dec 2020 17:22:29 +0000
parents e245a271fc44
children ccc8f0903d2e
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
1 #! /usr/bin/env python3
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
2
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
3 import psutil
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
4 import os
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
5 import os.path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
6 import rrdtool
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
7 import time
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
8 import subprocess
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
9
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
10 from pathlib import Path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
11
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
12 home = str(Path.home())
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
13 DB = os.path.join(home, ".load.rrd")
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
14
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
15 cpus = psutil.cpu_count()
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
16
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
17 config = [
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
18 ['load_1', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
19 ['load_5', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
20 ['load_15', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
21 *[[f'core{i+1}', 'GAUGE', 2, 0, 100] for i in range(cpus)],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
22 ['core_avg', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
23 ['GPU', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
24 ['user', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
25 ['system', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
26 ['iowait', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
27 ['mem_used', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
28 ['mem_buffers', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
29 ]
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
30
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
31 fields = len(config)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
32
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
33 def needs_creating():
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
34 if not os.path.exists(DB):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
35 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
36 else:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
37 cur_config = rrdtool.info(DB)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
38 for i, entry in enumerate(config):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
39 key, datatype, heartbeat, minval, maxval = entry
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
40 if f"ds[{key}].index" not in cur_config or \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
41 cur_config[f"ds[{key}].index"] != i:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
42 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
43 elif cur_config[f"ds[{key}].type"] != datatype or \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
44 cur_config[f"ds[{key}].minimal_heartbeat"] != heartbeat:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
45 # We don't appear to be able to check min/max from info
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
46 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
47 return False
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
48
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
49 # TODO: Add "pressure" support - relies on "psi=1" in kernel and /proc/pressure/… existing
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
50 if needs_creating():
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
51 rrdtool.create(DB, '--step', '1',
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
52 *[f'DS:{key}:{datatype}:{heartbeat}:{minval}:{maxval}' for \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
53 key, datatype, heartbeat, minval, maxval in config],
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
54 'RRA:AVERAGE:0.5:1:3600', #1hr of 1s interval (averaged)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
55 'RRA:MAX:0.5:60:360', #6hrs of 1 minute
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
56 'RRA:MAX:0.5:300:8640') #and 1mo of 5m resolution
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
57
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
58 samples = 10
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
59 gpu_idx = 3 + cpus + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
60 last_avg_idx = 3 + cpus + 2 # load + CPUs + CPU average + GPU
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
61 total_mem = psutil.virtual_memory().total
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
62
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
63 # Use dmon and assume we'll keep to roughly every second
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
64 # to keep up with its output to reduce the popen overhead
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
65 nv_smi = subprocess.Popen(['nvidia-smi', 'dmon', '-s', 'u'],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
66 stdout=subprocess.PIPE,
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
67 universal_newlines=True)
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
68
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
69 while True:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
70 interims = [[] for _ in range(fields)]
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
71 # Average 10 values internally to reduce RRD updates
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
72 # and to allow us to max some stats rather than average
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
73 for _ in range(0, 10):
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
74 cpu_pcs = psutil.cpu_percent(percpu=True)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
75 cpu_pc = sum(cpu_pcs) / cpus
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
76 #TODO: If cpu_pc > 25% (?) log top processes
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
77 cpu_states_pc = psutil.cpu_times_percent()
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
78 loads = os.getloadavg()
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
79 mem = psutil.virtual_memory()
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
80 i = 0
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
81 interims[i].append(loads[0])
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
82 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
83 interims[i].append(loads[1])
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
84 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
85 interims[i].append(loads[2])
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
86 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
87 for a_cpu_pc in cpu_pcs:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
88 interims[i].append(a_cpu_pc)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
89 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
90 interims[i].append(cpu_pc)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
91 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
92 interims[i].append(0) # Placeholder for GPU
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
93 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
94 interims[i].append(cpu_states_pc.user)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
95 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
96 interims[i].append(cpu_states_pc.system)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
97 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
98 interims[i].append(cpu_states_pc.iowait)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
99 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
100 interims[i].append((mem.used / total_mem) * 100)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
101 i = i + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
102 interims[i].append(((mem.buffers + mem.cached) / total_mem) * 100)
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
103 time.sleep(0.1)
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
104
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
105 vals = []
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
106 for i, interim_vals in enumerate(interims):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
107 if i < last_avg_idx:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
108 # Average most values
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
109 vals.append(sum(interim_vals) / 10)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
110 else:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
111 # But take the max CPU state value
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
112 vals.append(max(interim_vals))
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
113
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
114 while True:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
115 line = nv_smi.stdout.readline()
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
116 if line[0] != '#':
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
117 vals[gpu_idx] = int(line[8:11])
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
118 break
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
119
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
120 rrdtool.update(DB, "N:{}".format(':'.join(str(val) for val in vals)))