Mercurial > repos > other > usr-local-bin

annotate load-record @ 32:ccc8f0903d2e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use scheduling to keep better time Our previous approach just hoped that we kept to time, but long processing would slowly make our "sleep" periods wrong. This is especially likely at 0.1s (versus longer sleeps).
author IBBoard <dev@ibboard.co.uk>
date Sat, 27 Feb 2021 15:39:36 +0000
parents c2584db4a650
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
1 #! /usr/bin/env python3
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
2
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
3 import psutil
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
4 import os
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
5 import os.path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
6 import rrdtool
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
7 import sched
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
8 import subprocess
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
9 import threading
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
10
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
11 from pathlib import Path
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
12
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
13 home = str(Path.home())
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
14 DB = os.path.join(home, ".load.rrd")
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
15
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
16 cpus = psutil.cpu_count()
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
17
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
18 config = [
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
19 ['load_1', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
20 ['load_5', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
21 ['load_15', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
22 *[[f'core{i+1}', 'GAUGE', 2, 0, 100] for i in range(cpus)],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
23 ['core_avg', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
24 ['GPU', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
25 ['user', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
26 ['system', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
27 ['iowait', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
28 ['mem_used', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
29 ['mem_buffers', 'GAUGE', 2, 0, 100],
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
30 ]
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
31
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
32 fields = len(config)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
33
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
34
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
35 def needs_creating():
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
36 if not os.path.exists(DB):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
37 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
38 else:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
39 cur_config = rrdtool.info(DB)
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
40 for i, entry in enumerate(config):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
41 key, datatype, heartbeat, minval, maxval = entry
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
42 if f"ds[{key}].index" not in cur_config or \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
43 cur_config[f"ds[{key}].index"] != i:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
44 return True
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
45 elif cur_config[f"ds[{key}].type"] != datatype or \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
46 cur_config[f"ds[{key}].minimal_heartbeat"] != heartbeat:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
47 # We don't appear to be able to check min/max from info
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
48 return True
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
49 # TODO: Check RRA definitions based on rra[i].cf, rra[i].pdp_per_row and rra[i].rows
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
50 return False
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
51
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
52
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
53 # TODO: Add "pressure" support - relies on "psi=1" in kernel and /proc/pressure/… existing
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
54 if needs_creating():
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
55 rrdtool.create(DB, '--step', '1',
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
56 *[f'DS:{key}:{datatype}:{heartbeat}:{minval}:{maxval}' for \
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
57 key, datatype, heartbeat, minval, maxval in config],
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
58 'RRA:AVERAGE:0.5:1:3600', #1hr of 1s interval (averaged)
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
59 'RRA:MAX:0.5:60:360', #6hrs of 1 minute
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
60 'RRA:MAX:0.5:300:8640') #and 1mo of 5m resolution
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
61
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
62 samples = 10
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
63 gpu_idx = 3 + cpus + 1
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
64 last_avg_idx = 3 + cpus + 2 # load + CPUs + CPU average + GPU
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
65 total_mem = psutil.virtual_memory().total
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
66
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
67 # Note: We use some global variables on the assumption that:
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
68 # 1) We just need "the latest" gpu_val value
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
69 # 2) Because gpu_val is numeric then it can't be "inconsistent"
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
70 # 3) The use of the scheduler and its priorities ensures that
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
71 # the record_interims and record_record functions happen in sequence
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
72 # 4) The record_interims function takes under 1/samples seconds to run
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
73 #
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
74 # If this ever fails, we need to look at multiprocessing.Value and .Array
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
75 gpu_val = 0
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
76 interims = [[0] * samples for _ in range(fields)]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
77 pos = 0
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
78
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
79
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
80 def parse_nvidia_output():
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
81 global gpu_val
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
82 nv_smi = subprocess.Popen(['nvidia-smi', 'dmon', '-s', 'u'],
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
83 stdout=subprocess.PIPE,
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
84 universal_newlines=True)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
85 while True:
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
86 # Readline blocks, so this thread will update as and when new values are available
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
87 line = nv_smi.stdout.readline()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
88 if line and line[0] != '#':
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
89 gpu_val = int(line[8:11])
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
90
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
91
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
92 def record_interims():
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
93 global pos
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
94 scheduler.enter(1.0/samples, 1, record_interims)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
95 cur_pos = pos
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
96 pos = (pos + 1) % samples
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
97 cpu_pcs = psutil.cpu_percent(percpu=True)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
98 cpu_pc = sum(cpu_pcs) / cpus
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
99 #TODO: If cpu_pc > 25% (?) log top processes
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
100 cpu_states_pc = psutil.cpu_times_percent()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
101 loads = os.getloadavg()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
102 mem = psutil.virtual_memory()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
103 i = 0
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
104 interims[i][cur_pos] = loads[0]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
105 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
106 interims[i][cur_pos] = loads[1]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
107 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
108 interims[i][cur_pos] = loads[2]
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
109 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
110 for a_cpu_pc in cpu_pcs:
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
111 interims[i][cur_pos] = a_cpu_pc
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
112 i = i + 1
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
113 interims[i][cur_pos] = cpu_pc
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
114 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
115 interims[i][cur_pos] = 0 # Placeholder for GPU
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
116 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
117 interims[i][cur_pos] = cpu_states_pc.user
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
118 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
119 interims[i][cur_pos] = cpu_states_pc.system
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
120 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
121 interims[i][cur_pos] = cpu_states_pc.iowait
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
122 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
123 interims[i][cur_pos] = (mem.used / total_mem) * 100
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
124 i = i + 1
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
125 interims[i][cur_pos] = ((mem.buffers + mem.cached) / total_mem) * 100
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
126
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
127
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
128 def record_record():
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
129 global gpu_val, gpu_idx
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
130 scheduler.enter(1, 2, record_record)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
131 vals = [0] * fields
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
132 for i, interim_vals in enumerate(interims):
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
133 if i < last_avg_idx:
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
134 # Average most values
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
135 vals[i] = sum(interim_vals) / samples
19
e245a271fc44 Add scripts for recording/displaying CPU and GPU activity
IBBoard <dev@ibboard.co.uk>
parents:
diff changeset
136 else:
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
137 # But take the max CPU state value and memory usage
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
138 vals[i] = max(interim_vals)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
139
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
140 vals[gpu_idx] = gpu_val
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
141 rrdtool.update(DB, "N:{}".format(':'.join(str(val) for val in vals)))
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
142
29
c2584db4a650 Make load recording more flexible and efficient
IBBoard <dev@ibboard.co.uk>
parents: 19
diff changeset
143
32
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
144 nv_thread = threading.Thread(target=parse_nvidia_output)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
145 nv_thread.start()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
146 scheduler = sched.scheduler()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
147 # Let record_interims run and schedule itself
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
148 record_interims()
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
149 # Schedule record recording for 1s later so we've got a set of values, and then it'll schedule future calls
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
150 scheduler.enter(1, 2, record_record)
ccc8f0903d2e Use scheduling to keep better time
IBBoard <dev@ibboard.co.uk>
parents: 29
diff changeset
151 scheduler.run()