1 | #include <freebsd/machine/rtems-bsd-config.h> |
---|
2 | |
---|
3 | /*- |
---|
4 | * Copyright (c) 2001 Wind River Systems, Inc. |
---|
5 | * All rights reserved. |
---|
6 | * Written by: John Baldwin <jhb@FreeBSD.org> |
---|
7 | * |
---|
8 | * Copyright (c) 2009 Jeffrey Roberson <jeff@freebsd.org> |
---|
9 | * All rights reserved. |
---|
10 | * |
---|
11 | * Redistribution and use in source and binary forms, with or without |
---|
12 | * modification, are permitted provided that the following conditions |
---|
13 | * are met: |
---|
14 | * 1. Redistributions of source code must retain the above copyright |
---|
15 | * notice, this list of conditions and the following disclaimer. |
---|
16 | * 2. Redistributions in binary form must reproduce the above copyright |
---|
17 | * notice, this list of conditions and the following disclaimer in the |
---|
18 | * documentation and/or other materials provided with the distribution. |
---|
19 | * 4. Neither the name of the author nor the names of any co-contributors |
---|
20 | * may be used to endorse or promote products derived from this software |
---|
21 | * without specific prior written permission. |
---|
22 | * |
---|
23 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
---|
24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
---|
25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
---|
26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
---|
27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
---|
28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
---|
29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
---|
30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
---|
31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
---|
32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
---|
33 | * SUCH DAMAGE. |
---|
34 | */ |
---|
35 | |
---|
36 | /* |
---|
37 | * This module provides MI support for per-cpu data. |
---|
38 | * |
---|
39 | * Each architecture determines the mapping of logical CPU IDs to physical |
---|
40 | * CPUs. The requirements of this mapping are as follows: |
---|
41 | * - Logical CPU IDs must reside in the range 0 ... MAXCPU - 1. |
---|
42 | * - The mapping is not required to be dense. That is, there may be |
---|
43 | * gaps in the mappings. |
---|
44 | * - The platform sets the value of MAXCPU in <machine/param.h>. |
---|
45 | * - It is suggested, but not required, that in the non-SMP case, the |
---|
46 | * platform define MAXCPU to be 1 and define the logical ID of the |
---|
47 | * sole CPU as 0. |
---|
48 | */ |
---|
49 | |
---|
50 | #include <freebsd/sys/cdefs.h> |
---|
51 | __FBSDID("$FreeBSD$"); |
---|
52 | |
---|
53 | #include <freebsd/local/opt_ddb.h> |
---|
54 | |
---|
55 | #include <freebsd/sys/param.h> |
---|
56 | #include <freebsd/sys/systm.h> |
---|
57 | #include <freebsd/sys/sysctl.h> |
---|
58 | #include <freebsd/sys/linker_set.h> |
---|
59 | #include <freebsd/sys/lock.h> |
---|
60 | #include <freebsd/sys/malloc.h> |
---|
61 | #include <freebsd/sys/pcpu.h> |
---|
62 | #include <freebsd/sys/proc.h> |
---|
63 | #include <freebsd/sys/smp.h> |
---|
64 | #include <freebsd/sys/sx.h> |
---|
65 | #include <freebsd/ddb/ddb.h> |
---|
66 | |
---|
67 | MALLOC_DEFINE(M_PCPU, "Per-cpu", "Per-cpu resource accouting."); |
---|
68 | |
---|
69 | struct dpcpu_free { |
---|
70 | uintptr_t df_start; |
---|
71 | int df_len; |
---|
72 | TAILQ_ENTRY(dpcpu_free) df_link; |
---|
73 | }; |
---|
74 | |
---|
75 | static DPCPU_DEFINE(char, modspace[DPCPU_MODMIN]); |
---|
76 | static TAILQ_HEAD(, dpcpu_free) dpcpu_head = TAILQ_HEAD_INITIALIZER(dpcpu_head); |
---|
77 | static struct sx dpcpu_lock; |
---|
78 | uintptr_t dpcpu_off[MAXCPU]; |
---|
79 | struct pcpu *cpuid_to_pcpu[MAXCPU]; |
---|
80 | struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead); |
---|
81 | |
---|
82 | #ifndef __rtems__ |
---|
83 | /* |
---|
84 | * Initialize the MI portions of a struct pcpu. |
---|
85 | */ |
---|
86 | void |
---|
87 | pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) |
---|
88 | { |
---|
89 | |
---|
90 | bzero(pcpu, size); |
---|
91 | KASSERT(cpuid >= 0 && cpuid < MAXCPU, |
---|
92 | ("pcpu_init: invalid cpuid %d", cpuid)); |
---|
93 | pcpu->pc_cpuid = cpuid; |
---|
94 | pcpu->pc_cpumask = 1 << cpuid; |
---|
95 | cpuid_to_pcpu[cpuid] = pcpu; |
---|
96 | SLIST_INSERT_HEAD(&cpuhead, pcpu, pc_allcpu); |
---|
97 | cpu_pcpu_init(pcpu, cpuid, size); |
---|
98 | pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; |
---|
99 | pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; |
---|
100 | #ifdef KTR |
---|
101 | snprintf(pcpu->pc_name, sizeof(pcpu->pc_name), "CPU %d", cpuid); |
---|
102 | #endif |
---|
103 | } |
---|
104 | |
---|
105 | void |
---|
106 | dpcpu_init(void *dpcpu, int cpuid) |
---|
107 | { |
---|
108 | struct pcpu *pcpu; |
---|
109 | |
---|
110 | pcpu = pcpu_find(cpuid); |
---|
111 | pcpu->pc_dynamic = (uintptr_t)dpcpu - DPCPU_START; |
---|
112 | |
---|
113 | /* |
---|
114 | * Initialize defaults from our linker section. |
---|
115 | */ |
---|
116 | memcpy(dpcpu, (void *)DPCPU_START, DPCPU_BYTES); |
---|
117 | |
---|
118 | /* |
---|
119 | * Place it in the global pcpu offset array. |
---|
120 | */ |
---|
121 | dpcpu_off[cpuid] = pcpu->pc_dynamic; |
---|
122 | } |
---|
123 | |
---|
124 | static void |
---|
125 | dpcpu_startup(void *dummy __unused) |
---|
126 | { |
---|
127 | struct dpcpu_free *df; |
---|
128 | |
---|
129 | df = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); |
---|
130 | df->df_start = (uintptr_t)&DPCPU_NAME(modspace); |
---|
131 | df->df_len = DPCPU_MODMIN; |
---|
132 | TAILQ_INSERT_HEAD(&dpcpu_head, df, df_link); |
---|
133 | sx_init(&dpcpu_lock, "dpcpu alloc lock"); |
---|
134 | } |
---|
135 | SYSINIT(dpcpu, SI_SUB_KLD, SI_ORDER_FIRST, dpcpu_startup, 0); |
---|
136 | |
---|
137 | /* |
---|
138 | * First-fit extent based allocator for allocating space in the per-cpu |
---|
139 | * region reserved for modules. This is only intended for use by the |
---|
140 | * kernel linkers to place module linker sets. |
---|
141 | */ |
---|
142 | void * |
---|
143 | dpcpu_alloc(int size) |
---|
144 | { |
---|
145 | struct dpcpu_free *df; |
---|
146 | void *s; |
---|
147 | |
---|
148 | s = NULL; |
---|
149 | size = roundup2(size, sizeof(void *)); |
---|
150 | sx_xlock(&dpcpu_lock); |
---|
151 | TAILQ_FOREACH(df, &dpcpu_head, df_link) { |
---|
152 | if (df->df_len < size) |
---|
153 | continue; |
---|
154 | if (df->df_len == size) { |
---|
155 | s = (void *)df->df_start; |
---|
156 | TAILQ_REMOVE(&dpcpu_head, df, df_link); |
---|
157 | free(df, M_PCPU); |
---|
158 | break; |
---|
159 | } |
---|
160 | s = (void *)df->df_start; |
---|
161 | df->df_len -= size; |
---|
162 | df->df_start = df->df_start + size; |
---|
163 | break; |
---|
164 | } |
---|
165 | sx_xunlock(&dpcpu_lock); |
---|
166 | |
---|
167 | return (s); |
---|
168 | } |
---|
169 | |
---|
170 | /* |
---|
171 | * Free dynamic per-cpu space at module unload time. |
---|
172 | */ |
---|
173 | void |
---|
174 | dpcpu_free(void *s, int size) |
---|
175 | { |
---|
176 | struct dpcpu_free *df; |
---|
177 | struct dpcpu_free *dn; |
---|
178 | uintptr_t start; |
---|
179 | uintptr_t end; |
---|
180 | |
---|
181 | size = roundup2(size, sizeof(void *)); |
---|
182 | start = (uintptr_t)s; |
---|
183 | end = start + size; |
---|
184 | /* |
---|
185 | * Free a region of space and merge it with as many neighbors as |
---|
186 | * possible. Keeping the list sorted simplifies this operation. |
---|
187 | */ |
---|
188 | sx_xlock(&dpcpu_lock); |
---|
189 | TAILQ_FOREACH(df, &dpcpu_head, df_link) { |
---|
190 | if (df->df_start > end) |
---|
191 | break; |
---|
192 | /* |
---|
193 | * If we expand at the end of an entry we may have to |
---|
194 | * merge it with the one following it as well. |
---|
195 | */ |
---|
196 | if (df->df_start + df->df_len == start) { |
---|
197 | df->df_len += size; |
---|
198 | dn = TAILQ_NEXT(df, df_link); |
---|
199 | if (df->df_start + df->df_len == dn->df_start) { |
---|
200 | df->df_len += dn->df_len; |
---|
201 | TAILQ_REMOVE(&dpcpu_head, dn, df_link); |
---|
202 | free(dn, M_PCPU); |
---|
203 | } |
---|
204 | sx_xunlock(&dpcpu_lock); |
---|
205 | return; |
---|
206 | } |
---|
207 | if (df->df_start == end) { |
---|
208 | df->df_start = start; |
---|
209 | df->df_len += size; |
---|
210 | sx_xunlock(&dpcpu_lock); |
---|
211 | return; |
---|
212 | } |
---|
213 | } |
---|
214 | dn = malloc(sizeof(*df), M_PCPU, M_WAITOK | M_ZERO); |
---|
215 | dn->df_start = start; |
---|
216 | dn->df_len = size; |
---|
217 | if (df) |
---|
218 | TAILQ_INSERT_BEFORE(df, dn, df_link); |
---|
219 | else |
---|
220 | TAILQ_INSERT_TAIL(&dpcpu_head, dn, df_link); |
---|
221 | sx_xunlock(&dpcpu_lock); |
---|
222 | } |
---|
223 | |
---|
224 | /* |
---|
225 | * Initialize the per-cpu storage from an updated linker-set region. |
---|
226 | */ |
---|
227 | void |
---|
228 | dpcpu_copy(void *s, int size) |
---|
229 | { |
---|
230 | #ifdef SMP |
---|
231 | uintptr_t dpcpu; |
---|
232 | int i; |
---|
233 | |
---|
234 | for (i = 0; i < mp_ncpus; ++i) { |
---|
235 | dpcpu = dpcpu_off[i]; |
---|
236 | if (dpcpu == 0) |
---|
237 | continue; |
---|
238 | memcpy((void *)(dpcpu + (uintptr_t)s), s, size); |
---|
239 | } |
---|
240 | #else |
---|
241 | memcpy((void *)(dpcpu_off[0] + (uintptr_t)s), s, size); |
---|
242 | #endif |
---|
243 | } |
---|
244 | |
---|
245 | /* |
---|
246 | * Destroy a struct pcpu. |
---|
247 | */ |
---|
248 | void |
---|
249 | pcpu_destroy(struct pcpu *pcpu) |
---|
250 | { |
---|
251 | |
---|
252 | SLIST_REMOVE(&cpuhead, pcpu, pcpu, pc_allcpu); |
---|
253 | cpuid_to_pcpu[pcpu->pc_cpuid] = NULL; |
---|
254 | dpcpu_off[pcpu->pc_cpuid] = 0; |
---|
255 | } |
---|
256 | |
---|
257 | /* |
---|
258 | * Locate a struct pcpu by cpu id. |
---|
259 | */ |
---|
260 | struct pcpu * |
---|
261 | pcpu_find(u_int cpuid) |
---|
262 | { |
---|
263 | |
---|
264 | return (cpuid_to_pcpu[cpuid]); |
---|
265 | } |
---|
266 | |
---|
267 | int |
---|
268 | sysctl_dpcpu_quad(SYSCTL_HANDLER_ARGS) |
---|
269 | { |
---|
270 | uintptr_t dpcpu; |
---|
271 | int64_t count; |
---|
272 | int i; |
---|
273 | |
---|
274 | count = 0; |
---|
275 | for (i = 0; i < mp_ncpus; ++i) { |
---|
276 | dpcpu = dpcpu_off[i]; |
---|
277 | if (dpcpu == 0) |
---|
278 | continue; |
---|
279 | count += *(int64_t *)(dpcpu + (uintptr_t)arg1); |
---|
280 | } |
---|
281 | return (SYSCTL_OUT(req, &count, sizeof(count))); |
---|
282 | } |
---|
283 | |
---|
284 | int |
---|
285 | sysctl_dpcpu_long(SYSCTL_HANDLER_ARGS) |
---|
286 | { |
---|
287 | uintptr_t dpcpu; |
---|
288 | long count; |
---|
289 | int i; |
---|
290 | |
---|
291 | count = 0; |
---|
292 | for (i = 0; i < mp_ncpus; ++i) { |
---|
293 | dpcpu = dpcpu_off[i]; |
---|
294 | if (dpcpu == 0) |
---|
295 | continue; |
---|
296 | count += *(long *)(dpcpu + (uintptr_t)arg1); |
---|
297 | } |
---|
298 | return (SYSCTL_OUT(req, &count, sizeof(count))); |
---|
299 | } |
---|
300 | |
---|
301 | int |
---|
302 | sysctl_dpcpu_int(SYSCTL_HANDLER_ARGS) |
---|
303 | { |
---|
304 | uintptr_t dpcpu; |
---|
305 | int count; |
---|
306 | int i; |
---|
307 | |
---|
308 | count = 0; |
---|
309 | for (i = 0; i < mp_ncpus; ++i) { |
---|
310 | dpcpu = dpcpu_off[i]; |
---|
311 | if (dpcpu == 0) |
---|
312 | continue; |
---|
313 | count += *(int *)(dpcpu + (uintptr_t)arg1); |
---|
314 | } |
---|
315 | return (SYSCTL_OUT(req, &count, sizeof(count))); |
---|
316 | } |
---|
317 | |
---|
318 | #ifdef DDB |
---|
319 | DB_SHOW_COMMAND(dpcpu_off, db_show_dpcpu_off) |
---|
320 | { |
---|
321 | int id; |
---|
322 | |
---|
323 | for (id = 0; id <= mp_maxid; id++) { |
---|
324 | if (CPU_ABSENT(id)) |
---|
325 | continue; |
---|
326 | db_printf("dpcpu_off[%2d] = 0x%jx (+ DPCPU_START = %p)\n", |
---|
327 | id, (uintmax_t)dpcpu_off[id], |
---|
328 | (void *)(uintptr_t)(dpcpu_off[id] + DPCPU_START)); |
---|
329 | } |
---|
330 | } |
---|
331 | |
---|
332 | static void |
---|
333 | show_pcpu(struct pcpu *pc) |
---|
334 | { |
---|
335 | struct thread *td; |
---|
336 | |
---|
337 | db_printf("cpuid = %d\n", pc->pc_cpuid); |
---|
338 | db_printf("dynamic pcpu = %p\n", (void *)pc->pc_dynamic); |
---|
339 | db_printf("curthread = "); |
---|
340 | td = pc->pc_curthread; |
---|
341 | if (td != NULL) |
---|
342 | db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, |
---|
343 | td->td_name); |
---|
344 | else |
---|
345 | db_printf("none\n"); |
---|
346 | db_printf("curpcb = %p\n", pc->pc_curpcb); |
---|
347 | db_printf("fpcurthread = "); |
---|
348 | td = pc->pc_fpcurthread; |
---|
349 | if (td != NULL) |
---|
350 | db_printf("%p: pid %d \"%s\"\n", td, td->td_proc->p_pid, |
---|
351 | td->td_name); |
---|
352 | else |
---|
353 | db_printf("none\n"); |
---|
354 | db_printf("idlethread = "); |
---|
355 | td = pc->pc_idlethread; |
---|
356 | if (td != NULL) |
---|
357 | db_printf("%p: tid %d \"%s\"\n", td, td->td_tid, td->td_name); |
---|
358 | else |
---|
359 | db_printf("none\n"); |
---|
360 | db_show_mdpcpu(pc); |
---|
361 | |
---|
362 | #ifdef VIMAGE |
---|
363 | db_printf("curvnet = %p\n", pc->pc_curthread->td_vnet); |
---|
364 | #endif |
---|
365 | |
---|
366 | #ifdef WITNESS |
---|
367 | db_printf("spin locks held:\n"); |
---|
368 | witness_list_locks(&pc->pc_spinlocks, db_printf); |
---|
369 | #endif |
---|
370 | } |
---|
371 | |
---|
372 | DB_SHOW_COMMAND(pcpu, db_show_pcpu) |
---|
373 | { |
---|
374 | struct pcpu *pc; |
---|
375 | int id; |
---|
376 | |
---|
377 | if (have_addr) |
---|
378 | id = ((addr >> 4) % 16) * 10 + (addr % 16); |
---|
379 | else |
---|
380 | id = PCPU_GET(cpuid); |
---|
381 | pc = pcpu_find(id); |
---|
382 | if (pc == NULL) { |
---|
383 | db_printf("CPU %d not found\n", id); |
---|
384 | return; |
---|
385 | } |
---|
386 | show_pcpu(pc); |
---|
387 | } |
---|
388 | |
---|
389 | DB_SHOW_ALL_COMMAND(pcpu, db_show_cpu_all) |
---|
390 | { |
---|
391 | struct pcpu *pc; |
---|
392 | int id; |
---|
393 | |
---|
394 | db_printf("Current CPU: %d\n\n", PCPU_GET(cpuid)); |
---|
395 | for (id = 0; id <= mp_maxid; id++) { |
---|
396 | pc = pcpu_find(id); |
---|
397 | if (pc != NULL) { |
---|
398 | show_pcpu(pc); |
---|
399 | db_printf("\n"); |
---|
400 | } |
---|
401 | } |
---|
402 | } |
---|
403 | DB_SHOW_ALIAS(allpcpu, db_show_cpu_all); |
---|
404 | #endif |
---|
405 | #endif /* __rtems__ */ |
---|