Mercurial > repos > other > usr-local-bin

annotate load-record @ 38:737061eac1d0 default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Skip more large videos and some permission issues
author IBBoard <dev@ibboard.co.uk>
date Wed, 18 Aug 2021 20:40:17 +0100
parents ccc8f0903d2e
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
1 #! /usr/bin/env python3
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
2
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
3 import psutil
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
4 import os
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
5 import os.path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
6 import rrdtool
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
7 import sched
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
8 import subprocess
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
9 import threading
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
10
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
11 from pathlib import Path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
12
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
13 home = str(Path.home())
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
14 DB = os.path.join(home, ".load.rrd")
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
15
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
16 cpus = psutil.cpu_count()
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
17
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
18 config = [
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
19 ['load_1', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
20 ['load_5', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
21 ['load_15', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
22 *[[f'core{i+1}', 'GAUGE', 2, 0, 100] for i in range(cpus)],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
23 ['core_avg', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
24 ['GPU', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
25 ['user', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
26 ['system', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
27 ['iowait', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
28 ['mem_used', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
29 ['mem_buffers', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
30 ]
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
31
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
32 fields = len(config)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
33
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
34
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
35 def needs_creating():
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
36 if not os.path.exists(DB):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
37 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
38 else:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
39 cur_config = rrdtool.info(DB)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
40 for i, entry in enumerate(config):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
41 key, datatype, heartbeat, minval, maxval = entry
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
42 if f"ds[{key}].index" not in cur_config or \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
43 cur_config[f"ds[{key}].index"] != i:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
44 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
45 elif cur_config[f"ds[{key}].type"] != datatype or \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
46 cur_config[f"ds[{key}].minimal_heartbeat"] != heartbeat:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
47 # We don't appear to be able to check min/max from info
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
48 return True
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
49 # TODO: Check RRA definitions based on rra[i].cf, rra[i].pdp_per_row and rra[i].rows
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
50 return False
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
51
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
52
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
53 # TODO: Add "pressure" support - relies on "psi=1" in kernel and /proc/pressure/… existing
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
54 if needs_creating():
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
55 rrdtool.create(DB, '--step', '1',
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
56 *[f'DS:{key}:{datatype}:{heartbeat}:{minval}:{maxval}' for \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
57 key, datatype, heartbeat, minval, maxval in config],
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
58 'RRA:AVERAGE:0.5:1:3600', #1hr of 1s interval (averaged)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
59 'RRA:MAX:0.5:60:360', #6hrs of 1 minute
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
60 'RRA:MAX:0.5:300:8640') #and 1mo of 5m resolution
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
61
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
62 samples = 10
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
63 gpu_idx = 3 + cpus + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
64 last_avg_idx = 3 + cpus + 2 # load + CPUs + CPU average + GPU
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
65 total_mem = psutil.virtual_memory().total
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
66
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
67 # Note: We use some global variables on the assumption that:
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
68 # 1) We just need "the latest" gpu_val value
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
69 # 2) Because gpu_val is numeric then it can't be "inconsistent"
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
70 # 3) The use of the scheduler and its priorities ensures that
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
71 # the record_interims and record_record functions happen in sequence
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
72 # 4) The record_interims function takes under 1/samples seconds to run
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
73 #
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
74 # If this ever fails, we need to look at multiprocessing.Value and .Array
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
75 gpu_val = 0
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
76 interims = [[0] * samples for _ in range(fields)]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
77 pos = 0
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
78
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
79
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
80 def parse_nvidia_output():
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
81 global gpu_val
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
82 nv_smi = subprocess.Popen(['nvidia-smi', 'dmon', '-s', 'u'],
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
83 stdout=subprocess.PIPE,
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
84 universal_newlines=True)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
85 while True:
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
86 # Readline blocks, so this thread will update as and when new values are available
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
87 line = nv_smi.stdout.readline()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
88 if line and line[0] != '#':
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
89 gpu_val = int(line[8:11])
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
90
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
91
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
92 def record_interims():
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
93 global pos
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
94 scheduler.enter(1.0/samples, 1, record_interims)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
95 cur_pos = pos
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
96 pos = (pos + 1) % samples
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
97 cpu_pcs = psutil.cpu_percent(percpu=True)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
98 cpu_pc = sum(cpu_pcs) / cpus
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
99 #TODO: If cpu_pc > 25% (?) log top processes
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
100 cpu_states_pc = psutil.cpu_times_percent()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
101 loads = os.getloadavg()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
102 mem = psutil.virtual_memory()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
103 i = 0
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
104 interims[i][cur_pos] = loads[0]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
105 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
106 interims[i][cur_pos] = loads[1]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
107 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
108 interims[i][cur_pos] = loads[2]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
109 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
110 for a_cpu_pc in cpu_pcs:
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
111 interims[i][cur_pos] = a_cpu_pc
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
112 i = i + 1
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
113 interims[i][cur_pos] = cpu_pc
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
114 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
115 interims[i][cur_pos] = 0 # Placeholder for GPU
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
116 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
117 interims[i][cur_pos] = cpu_states_pc.user
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
118 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
119 interims[i][cur_pos] = cpu_states_pc.system
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
120 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
121 interims[i][cur_pos] = cpu_states_pc.iowait
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
122 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
123 interims[i][cur_pos] = (mem.used / total_mem) * 100
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
124 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
125 interims[i][cur_pos] = ((mem.buffers + mem.cached) / total_mem) * 100
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
126
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
127
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
128 def record_record():
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
129 global gpu_val, gpu_idx
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
130 scheduler.enter(1, 2, record_record)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
131 vals = [0] * fields
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
132 for i, interim_vals in enumerate(interims):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
133 if i < last_avg_idx:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
134 # Average most values
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
135 vals[i] = sum(interim_vals) / samples
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
136 else:
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
137 # But take the max CPU state value and memory usage
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
138 vals[i] = max(interim_vals)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
139
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
140 vals[gpu_idx] = gpu_val
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
141 rrdtool.update(DB, "N:{}".format(':'.join(str(val) for val in vals)))
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
142
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
143
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
144 nv_thread = threading.Thread(target=parse_nvidia_output)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
145 nv_thread.start()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
146 scheduler = sched.scheduler()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
147 # Let record_interims run and schedule itself
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
148 record_interims()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
149 # Schedule record recording for 1s later so we've got a set of values, and then it'll schedule future calls
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
150 scheduler.enter(1, 2, record_record)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
151 scheduler.run()