Mercurial > repos > other > usr-local-bin

annotate load-record @ 27:c7621908cb47

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Try to manage disk usage for backup
author IBBoard <dev@ibboard.co.uk>
date Sat, 07 Mar 2020 16:37:38 +0000
parents e245a271fc44
children c2584db4a650
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
1 #! /usr/bin/env python3
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
2
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
3 import psutil
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
4 import os
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
5 import os.path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
6 import rrdtool
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
7 import time
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
8 import subprocess
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
9
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
10 from pathlib import Path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
11
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
12 home = str(Path.home())
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
13 DB = os.path.join(home, ".load.rrd")
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
14
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
15 if not os.path.exists(DB):
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
16 rrdtool.create(DB, '--step', '1',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
17 'DS:load_1:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
18 'DS:load_5:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
19 'DS:load_15:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
20 'DS:core1:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
21 'DS:core2:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
22 'DS:core3:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
23 'DS:core4:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
24 'DS:core_avg:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
25 'DS:GPU:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
26 'DS:user:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
27 'DS:system:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
28 'DS:iowait:GAUGE:2:0.0:100.0',
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
29 'RRA:AVERAGE:0.5:1:3600', #1hr of 1s interval (averaged)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
30 'RRA:MAX:0.5:60:360', #6hrs of 1 minute
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
31 'RRA:MAX:0.5:300:8640') #and 1mo of 5m resolution
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
32
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
33 cpus = psutil.cpu_count()
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
34 fields = 12
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
35
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
36 while True:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
37 interims = [[] for _ in range(fields)]
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
38 for _ in range(0, 10):
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
39 cpu_pcs = psutil.cpu_percent(percpu=True)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
40 cpu_pc = sum(cpu_pcs) / cpus
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
41 #TODO: If cpu_pc > 25% (?) log top processes
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
42 cpu_states_pc = psutil.cpu_times_percent()
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
43 loads = os.getloadavg()
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
44 # TODO: Include GPU load
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
45 nv_smi = subprocess.run(["nvidia-smi", "stats", "-d", "gpuUtil", "-c", "1"], stdout=subprocess.PIPE, universal_newlines=True)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
46 gpu = 0
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
47 lines = nv_smi.stdout.split('\n')
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
48 line_count = len(lines)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
49 for line in lines:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
50 line_parts = line.split(',')
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
51 if len(line_parts) == 4:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
52 gpu += int(line_parts[3])
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
53 else:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
54 line_count -= 1
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
55 gpu = gpu / line_count
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
56 time.sleep(0.1)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
57 interims[0].append(loads[0])
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
58 interims[1].append(loads[1])
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
59 interims[2].append(loads[2])
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
60 for i in range(cpus):
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
61 interims[i+3].append(cpu_pcs[i])
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
62 i = cpus + 3
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
63 interims[i].append(cpu_pc)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
64 i += 1
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
65 interims[i].append(gpu)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
66 i += 1
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
67 interims[i].append(cpu_states_pc.user)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
68 i += 1
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
69 interims[i].append(cpu_states_pc.system)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
70 i += 1
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
71 interims[i].append(cpu_states_pc.iowait)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
72 vals = []
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
73 for i in range(fields):
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
74 interim_vals = interims[i]
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
75 # Average most values
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
76 if i < fields - 3:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
77 vals.append(sum(interim_vals) / 10)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
78 else:
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
79 # But take the max CPU state value
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
80 vals.append(max(interim_vals))
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
81 rrdtool.update(DB, "N:{}".format(':'.join(str(val) for val in vals)))