1 | #include <freebsd/machine/rtems-bsd-config.h> |
---|
2 | |
---|
3 | /*- |
---|
4 | * Copyright (c) 2007-2009 Robert N. M. Watson |
---|
5 | * All rights reserved. |
---|
6 | * |
---|
7 | * Redistribution and use in source and binary forms, with or without |
---|
8 | * modification, are permitted provided that the following conditions |
---|
9 | * are met: |
---|
10 | * 1. Redistributions of source code must retain the above copyright |
---|
11 | * notice, this list of conditions and the following disclaimer. |
---|
12 | * 2. Redistributions in binary form must reproduce the above copyright |
---|
13 | * notice, this list of conditions and the following disclaimer in the |
---|
14 | * documentation and/or other materials provided with the distribution. |
---|
15 | * |
---|
16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
---|
17 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
---|
18 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
---|
19 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
---|
20 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
---|
21 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
---|
22 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
---|
23 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
---|
24 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
---|
25 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
---|
26 | * SUCH DAMAGE. |
---|
27 | */ |
---|
28 | |
---|
29 | #include <freebsd/sys/cdefs.h> |
---|
30 | __FBSDID("$FreeBSD$"); |
---|
31 | |
---|
32 | /* |
---|
33 | * netisr is a packet dispatch service, allowing synchronous (directly |
---|
34 | * dispatched) and asynchronous (deferred dispatch) processing of packets by |
---|
35 | * registered protocol handlers. Callers pass a protocol identifier and |
---|
36 | * packet to netisr, along with a direct dispatch hint, and work will either |
---|
37 | * be immediately processed with the registered handler, or passed to a |
---|
38 | * kernel software interrupt (SWI) thread for deferred dispatch. Callers |
---|
39 | * will generally select one or the other based on: |
---|
40 | * |
---|
41 | * - Might directly dispatching a netisr handler lead to code reentrance or |
---|
42 | * lock recursion, such as entering the socket code from the socket code. |
---|
43 | * - Might directly dispatching a netisr handler lead to recursive |
---|
44 | * processing, such as when decapsulating several wrapped layers of tunnel |
---|
45 | * information (IPSEC within IPSEC within ...). |
---|
46 | * |
---|
47 | * Maintaining ordering for protocol streams is a critical design concern. |
---|
48 | * Enforcing ordering limits the opportunity for concurrency, but maintains |
---|
49 | * the strong ordering requirements found in some protocols, such as TCP. Of |
---|
50 | * related concern is CPU affinity--it is desirable to process all data |
---|
51 | * associated with a particular stream on the same CPU over time in order to |
---|
52 | * avoid acquiring locks associated with the connection on different CPUs, |
---|
53 | * keep connection data in one cache, and to generally encourage associated |
---|
54 | * user threads to live on the same CPU as the stream. It's also desirable |
---|
55 | * to avoid lock migration and contention where locks are associated with |
---|
56 | * more than one flow. |
---|
57 | * |
---|
58 | * netisr supports several policy variations, represented by the |
---|
59 | * NETISR_POLICY_* constants, allowing protocols to play a varying role in |
---|
60 | * identifying flows, assigning work to CPUs, etc. These are described in |
---|
61 | * detail in netisr.h. |
---|
62 | */ |
---|
63 | |
---|
64 | #include <freebsd/local/opt_ddb.h> |
---|
65 | #include <freebsd/local/opt_device_polling.h> |
---|
66 | |
---|
67 | #include <freebsd/sys/param.h> |
---|
68 | #include <freebsd/sys/bus.h> |
---|
69 | #include <freebsd/sys/kernel.h> |
---|
70 | #include <freebsd/sys/kthread.h> |
---|
71 | #include <freebsd/sys/interrupt.h> |
---|
72 | #include <freebsd/sys/lock.h> |
---|
73 | #include <freebsd/sys/mbuf.h> |
---|
74 | #include <freebsd/sys/mutex.h> |
---|
75 | #include <freebsd/sys/pcpu.h> |
---|
76 | #include <freebsd/sys/proc.h> |
---|
77 | #include <freebsd/sys/rmlock.h> |
---|
78 | #include <freebsd/sys/sched.h> |
---|
79 | #include <freebsd/sys/smp.h> |
---|
80 | #include <freebsd/sys/socket.h> |
---|
81 | #include <freebsd/sys/sysctl.h> |
---|
82 | #include <freebsd/sys/systm.h> |
---|
83 | |
---|
84 | #ifdef DDB |
---|
85 | #include <freebsd/ddb/ddb.h> |
---|
86 | #endif |
---|
87 | |
---|
88 | #include <freebsd/net/if.h> |
---|
89 | #include <freebsd/net/if_var.h> |
---|
90 | #include <freebsd/net/netisr.h> |
---|
91 | #include <freebsd/net/vnet.h> |
---|
92 | |
---|
93 | /*- |
---|
94 | * Synchronize use and modification of the registered netisr data structures; |
---|
95 | * acquire a read lock while modifying the set of registered protocols to |
---|
96 | * prevent partially registered or unregistered protocols from being run. |
---|
97 | * |
---|
98 | * The following data structures and fields are protected by this lock: |
---|
99 | * |
---|
100 | * - The np array, including all fields of struct netisr_proto. |
---|
101 | * - The nws array, including all fields of struct netisr_worker. |
---|
102 | * - The nws_array array. |
---|
103 | * |
---|
104 | * Note: the NETISR_LOCKING define controls whether read locks are acquired |
---|
105 | * in packet processing paths requiring netisr registration stability. This |
---|
106 | * is disabled by default as it can lead to a measurable performance |
---|
107 | * degradation even with rmlocks (3%-6% for loopback ping-pong traffic), and |
---|
108 | * because netisr registration and unregistration is extremely rare at |
---|
109 | * runtime. If it becomes more common, this decision should be revisited. |
---|
110 | * |
---|
111 | * XXXRW: rmlocks don't support assertions. |
---|
112 | */ |
---|
113 | static struct rmlock netisr_rmlock; |
---|
114 | #define NETISR_LOCK_INIT() rm_init_flags(&netisr_rmlock, "netisr", \ |
---|
115 | RM_NOWITNESS) |
---|
116 | #define NETISR_LOCK_ASSERT() |
---|
117 | #define NETISR_RLOCK(tracker) rm_rlock(&netisr_rmlock, (tracker)) |
---|
118 | #define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker)) |
---|
119 | #define NETISR_WLOCK() rm_wlock(&netisr_rmlock) |
---|
120 | #define NETISR_WUNLOCK() rm_wunlock(&netisr_rmlock) |
---|
121 | /* #define NETISR_LOCKING */ |
---|
122 | |
---|
123 | SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr"); |
---|
124 | |
---|
125 | /*- |
---|
126 | * Three direct dispatch policies are supported: |
---|
127 | * |
---|
128 | * - Always defer: all work is scheduled for a netisr, regardless of context. |
---|
129 | * (!direct) |
---|
130 | * |
---|
131 | * - Hybrid: if the executing context allows direct dispatch, and we're |
---|
132 | * running on the CPU the work would be done on, then direct dispatch if it |
---|
133 | * wouldn't violate ordering constraints on the workstream. |
---|
134 | * (direct && !direct_force) |
---|
135 | * |
---|
136 | * - Always direct: if the executing context allows direct dispatch, always |
---|
137 | * direct dispatch. (direct && direct_force) |
---|
138 | * |
---|
139 | * Notice that changing the global policy could lead to short periods of |
---|
140 | * misordered processing, but this is considered acceptable as compared to |
---|
141 | * the complexity of enforcing ordering during policy changes. |
---|
142 | */ |
---|
143 | static int netisr_direct_force = 1; /* Always direct dispatch. */ |
---|
144 | TUNABLE_INT("net.isr.direct_force", &netisr_direct_force); |
---|
145 | SYSCTL_INT(_net_isr, OID_AUTO, direct_force, CTLFLAG_RW, |
---|
146 | &netisr_direct_force, 0, "Force direct dispatch"); |
---|
147 | |
---|
148 | static int netisr_direct = 1; /* Enable direct dispatch. */ |
---|
149 | TUNABLE_INT("net.isr.direct", &netisr_direct); |
---|
150 | SYSCTL_INT(_net_isr, OID_AUTO, direct, CTLFLAG_RW, |
---|
151 | &netisr_direct, 0, "Enable direct dispatch"); |
---|
152 | |
---|
153 | /* |
---|
154 | * Allow the administrator to limit the number of threads (CPUs) to use for |
---|
155 | * netisr. We don't check netisr_maxthreads before creating the thread for |
---|
156 | * CPU 0, so in practice we ignore values <= 1. This must be set at boot. |
---|
157 | * We will create at most one thread per CPU. |
---|
158 | */ |
---|
159 | static int netisr_maxthreads = -1; /* Max number of threads. */ |
---|
160 | TUNABLE_INT("net.isr.maxthreads", &netisr_maxthreads); |
---|
161 | SYSCTL_INT(_net_isr, OID_AUTO, maxthreads, CTLFLAG_RD, |
---|
162 | &netisr_maxthreads, 0, |
---|
163 | "Use at most this many CPUs for netisr processing"); |
---|
164 | |
---|
165 | static int netisr_bindthreads = 0; /* Bind threads to CPUs. */ |
---|
166 | TUNABLE_INT("net.isr.bindthreads", &netisr_bindthreads); |
---|
167 | SYSCTL_INT(_net_isr, OID_AUTO, bindthreads, CTLFLAG_RD, |
---|
168 | &netisr_bindthreads, 0, "Bind netisr threads to CPUs."); |
---|
169 | |
---|
170 | /* |
---|
171 | * Limit per-workstream queues to at most net.isr.maxqlimit, both for initial |
---|
172 | * configuration and later modification using netisr_setqlimit(). |
---|
173 | */ |
---|
174 | #define NETISR_DEFAULT_MAXQLIMIT 10240 |
---|
175 | static u_int netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT; |
---|
176 | TUNABLE_INT("net.isr.maxqlimit", &netisr_maxqlimit); |
---|
177 | SYSCTL_INT(_net_isr, OID_AUTO, maxqlimit, CTLFLAG_RD, |
---|
178 | &netisr_maxqlimit, 0, |
---|
179 | "Maximum netisr per-protocol, per-CPU queue depth."); |
---|
180 | |
---|
181 | /* |
---|
182 | * The default per-workstream queue limit for protocols that don't initialize |
---|
183 | * the nh_qlimit field of their struct netisr_handler. If this is set above |
---|
184 | * netisr_maxqlimit, we truncate it to the maximum during boot. |
---|
185 | */ |
---|
186 | #define NETISR_DEFAULT_DEFAULTQLIMIT 256 |
---|
187 | static u_int netisr_defaultqlimit = NETISR_DEFAULT_DEFAULTQLIMIT; |
---|
188 | TUNABLE_INT("net.isr.defaultqlimit", &netisr_defaultqlimit); |
---|
189 | SYSCTL_INT(_net_isr, OID_AUTO, defaultqlimit, CTLFLAG_RD, |
---|
190 | &netisr_defaultqlimit, 0, |
---|
191 | "Default netisr per-protocol, per-CPU queue limit if not set by protocol"); |
---|
192 | |
---|
193 | /* |
---|
194 | * Each protocol is described by a struct netisr_proto, which holds all |
---|
195 | * global per-protocol information. This data structure is set up by |
---|
196 | * netisr_register(), and derived from the public struct netisr_handler. |
---|
197 | */ |
---|
198 | struct netisr_proto { |
---|
199 | const char *np_name; /* Character string protocol name. */ |
---|
200 | netisr_handler_t *np_handler; /* Protocol handler. */ |
---|
201 | netisr_m2flow_t *np_m2flow; /* Query flow for untagged packet. */ |
---|
202 | netisr_m2cpuid_t *np_m2cpuid; /* Query CPU to process packet on. */ |
---|
203 | netisr_drainedcpu_t *np_drainedcpu; /* Callback when drained a queue. */ |
---|
204 | u_int np_qlimit; /* Maximum per-CPU queue depth. */ |
---|
205 | u_int np_policy; /* Work placement policy. */ |
---|
206 | }; |
---|
207 | |
---|
208 | #define NETISR_MAXPROT 16 /* Compile-time limit. */ |
---|
209 | |
---|
210 | /* |
---|
211 | * The np array describes all registered protocols, indexed by protocol |
---|
212 | * number. |
---|
213 | */ |
---|
214 | static struct netisr_proto np[NETISR_MAXPROT]; |
---|
215 | |
---|
216 | /* |
---|
217 | * Protocol-specific work for each workstream is described by struct |
---|
218 | * netisr_work. Each work descriptor consists of an mbuf queue and |
---|
219 | * statistics. |
---|
220 | */ |
---|
221 | struct netisr_work { |
---|
222 | /* |
---|
223 | * Packet queue, linked by m_nextpkt. |
---|
224 | */ |
---|
225 | struct mbuf *nw_head; |
---|
226 | struct mbuf *nw_tail; |
---|
227 | u_int nw_len; |
---|
228 | u_int nw_qlimit; |
---|
229 | u_int nw_watermark; |
---|
230 | |
---|
231 | /* |
---|
232 | * Statistics -- written unlocked, but mostly from curcpu. |
---|
233 | */ |
---|
234 | u_int64_t nw_dispatched; /* Number of direct dispatches. */ |
---|
235 | u_int64_t nw_hybrid_dispatched; /* "" hybrid dispatches. */ |
---|
236 | u_int64_t nw_qdrops; /* "" drops. */ |
---|
237 | u_int64_t nw_queued; /* "" enqueues. */ |
---|
238 | u_int64_t nw_handled; /* "" handled in worker. */ |
---|
239 | }; |
---|
240 | |
---|
241 | /* |
---|
242 | * Workstreams hold a set of ordered work across each protocol, and are |
---|
243 | * described by netisr_workstream. Each workstream is associated with a |
---|
244 | * worker thread, which in turn is pinned to a CPU. Work associated with a |
---|
245 | * workstream can be processd in other threads during direct dispatch; |
---|
246 | * concurrent processing is prevented by the NWS_RUNNING flag, which |
---|
247 | * indicates that a thread is already processing the work queue. |
---|
248 | */ |
---|
249 | struct netisr_workstream { |
---|
250 | struct intr_event *nws_intr_event; /* Handler for stream. */ |
---|
251 | void *nws_swi_cookie; /* swi(9) cookie for stream. */ |
---|
252 | struct mtx nws_mtx; /* Synchronize work. */ |
---|
253 | u_int nws_cpu; /* CPU pinning. */ |
---|
254 | u_int nws_flags; /* Wakeup flags. */ |
---|
255 | u_int nws_pendingbits; /* Scheduled protocols. */ |
---|
256 | |
---|
257 | /* |
---|
258 | * Each protocol has per-workstream data. |
---|
259 | */ |
---|
260 | struct netisr_work nws_work[NETISR_MAXPROT]; |
---|
261 | } __aligned(CACHE_LINE_SIZE); |
---|
262 | |
---|
263 | /* |
---|
264 | * Per-CPU workstream data. |
---|
265 | */ |
---|
266 | DPCPU_DEFINE(struct netisr_workstream, nws); |
---|
267 | |
---|
268 | /* |
---|
269 | * Map contiguous values between 0 and nws_count into CPU IDs appropriate for |
---|
270 | * accessing workstreams. This allows constructions of the form |
---|
271 | * DPCPU_ID_GET(nws_array[arbitraryvalue % nws_count], nws). |
---|
272 | */ |
---|
273 | static u_int nws_array[MAXCPU]; |
---|
274 | |
---|
275 | /* |
---|
276 | * Number of registered workstreams. Will be at most the number of running |
---|
277 | * CPUs once fully started. |
---|
278 | */ |
---|
279 | static u_int nws_count; |
---|
280 | SYSCTL_INT(_net_isr, OID_AUTO, numthreads, CTLFLAG_RD, |
---|
281 | &nws_count, 0, "Number of extant netisr threads."); |
---|
282 | |
---|
283 | /* |
---|
284 | * Per-workstream flags. |
---|
285 | */ |
---|
286 | #define NWS_RUNNING 0x00000001 /* Currently running in a thread. */ |
---|
287 | #define NWS_DISPATCHING 0x00000002 /* Currently being direct-dispatched. */ |
---|
288 | #define NWS_SCHEDULED 0x00000004 /* Signal issued. */ |
---|
289 | |
---|
290 | /* |
---|
291 | * Synchronization for each workstream: a mutex protects all mutable fields |
---|
292 | * in each stream, including per-protocol state (mbuf queues). The SWI is |
---|
293 | * woken up if asynchronous dispatch is required. |
---|
294 | */ |
---|
295 | #define NWS_LOCK(s) mtx_lock(&(s)->nws_mtx) |
---|
296 | #define NWS_LOCK_ASSERT(s) mtx_assert(&(s)->nws_mtx, MA_OWNED) |
---|
297 | #define NWS_UNLOCK(s) mtx_unlock(&(s)->nws_mtx) |
---|
298 | #define NWS_SIGNAL(s) swi_sched((s)->nws_swi_cookie, 0) |
---|
299 | |
---|
300 | #ifndef __rtems__ |
---|
301 | /* |
---|
302 | * Utility routines for protocols that implement their own mapping of flows |
---|
303 | * to CPUs. |
---|
304 | */ |
---|
305 | u_int |
---|
306 | netisr_get_cpucount(void) |
---|
307 | { |
---|
308 | |
---|
309 | return (nws_count); |
---|
310 | } |
---|
311 | |
---|
312 | u_int |
---|
313 | netisr_get_cpuid(u_int cpunumber) |
---|
314 | { |
---|
315 | |
---|
316 | KASSERT(cpunumber < nws_count, ("%s: %u > %u", __func__, cpunumber, |
---|
317 | nws_count)); |
---|
318 | |
---|
319 | return (nws_array[cpunumber]); |
---|
320 | } |
---|
321 | #endif /* __rtems__ */ |
---|
322 | |
---|
323 | /* |
---|
324 | * The default implementation of -> CPU ID mapping. |
---|
325 | * |
---|
326 | * Non-static so that protocols can use it to map their own work to specific |
---|
327 | * CPUs in a manner consistent to netisr for affinity purposes. |
---|
328 | */ |
---|
329 | u_int |
---|
330 | netisr_default_flow2cpu(u_int flowid) |
---|
331 | { |
---|
332 | |
---|
333 | return (nws_array[flowid % nws_count]); |
---|
334 | } |
---|
335 | |
---|
336 | /* |
---|
337 | * Register a new netisr handler, which requires initializing per-protocol |
---|
338 | * fields for each workstream. All netisr work is briefly suspended while |
---|
339 | * the protocol is installed. |
---|
340 | */ |
---|
341 | void |
---|
342 | netisr_register(const struct netisr_handler *nhp) |
---|
343 | { |
---|
344 | struct netisr_work *npwp; |
---|
345 | const char *name; |
---|
346 | u_int i, proto; |
---|
347 | |
---|
348 | proto = nhp->nh_proto; |
---|
349 | name = nhp->nh_name; |
---|
350 | |
---|
351 | /* |
---|
352 | * Test that the requested registration is valid. |
---|
353 | */ |
---|
354 | KASSERT(nhp->nh_name != NULL, |
---|
355 | ("%s: nh_name NULL for %u", __func__, proto)); |
---|
356 | KASSERT(nhp->nh_handler != NULL, |
---|
357 | ("%s: nh_handler NULL for %s", __func__, name)); |
---|
358 | KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE || |
---|
359 | nhp->nh_policy == NETISR_POLICY_FLOW || |
---|
360 | nhp->nh_policy == NETISR_POLICY_CPU, |
---|
361 | ("%s: unsupported nh_policy %u for %s", __func__, |
---|
362 | nhp->nh_policy, name)); |
---|
363 | KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW || |
---|
364 | nhp->nh_m2flow == NULL, |
---|
365 | ("%s: nh_policy != FLOW but m2flow defined for %s", __func__, |
---|
366 | name)); |
---|
367 | KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpuid == NULL, |
---|
368 | ("%s: nh_policy != CPU but m2cpuid defined for %s", __func__, |
---|
369 | name)); |
---|
370 | KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpuid != NULL, |
---|
371 | ("%s: nh_policy == CPU but m2cpuid not defined for %s", __func__, |
---|
372 | name)); |
---|
373 | KASSERT(proto < NETISR_MAXPROT, |
---|
374 | ("%s(%u, %s): protocol too big", __func__, proto, name)); |
---|
375 | |
---|
376 | /* |
---|
377 | * Test that no existing registration exists for this protocol. |
---|
378 | */ |
---|
379 | NETISR_WLOCK(); |
---|
380 | KASSERT(np[proto].np_name == NULL, |
---|
381 | ("%s(%u, %s): name present", __func__, proto, name)); |
---|
382 | KASSERT(np[proto].np_handler == NULL, |
---|
383 | ("%s(%u, %s): handler present", __func__, proto, name)); |
---|
384 | |
---|
385 | np[proto].np_name = name; |
---|
386 | np[proto].np_handler = nhp->nh_handler; |
---|
387 | np[proto].np_m2flow = nhp->nh_m2flow; |
---|
388 | np[proto].np_m2cpuid = nhp->nh_m2cpuid; |
---|
389 | np[proto].np_drainedcpu = nhp->nh_drainedcpu; |
---|
390 | if (nhp->nh_qlimit == 0) |
---|
391 | np[proto].np_qlimit = netisr_defaultqlimit; |
---|
392 | else if (nhp->nh_qlimit > netisr_maxqlimit) { |
---|
393 | printf("%s: %s requested queue limit %u capped to " |
---|
394 | "net.isr.maxqlimit %u\n", __func__, name, nhp->nh_qlimit, |
---|
395 | netisr_maxqlimit); |
---|
396 | np[proto].np_qlimit = netisr_maxqlimit; |
---|
397 | } else |
---|
398 | np[proto].np_qlimit = nhp->nh_qlimit; |
---|
399 | np[proto].np_policy = nhp->nh_policy; |
---|
400 | for (i = 0; i <= mp_maxid; i++) { |
---|
401 | if (CPU_ABSENT(i)) |
---|
402 | continue; |
---|
403 | npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; |
---|
404 | bzero(npwp, sizeof(*npwp)); |
---|
405 | npwp->nw_qlimit = np[proto].np_qlimit; |
---|
406 | } |
---|
407 | NETISR_WUNLOCK(); |
---|
408 | } |
---|
409 | |
---|
410 | /* |
---|
411 | * Clear drop counters across all workstreams for a protocol. |
---|
412 | */ |
---|
413 | void |
---|
414 | netisr_clearqdrops(const struct netisr_handler *nhp) |
---|
415 | { |
---|
416 | struct netisr_work *npwp; |
---|
417 | #ifdef INVARIANTS |
---|
418 | const char *name; |
---|
419 | #endif |
---|
420 | u_int i, proto; |
---|
421 | |
---|
422 | proto = nhp->nh_proto; |
---|
423 | #ifdef INVARIANTS |
---|
424 | name = nhp->nh_name; |
---|
425 | #endif |
---|
426 | KASSERT(proto < NETISR_MAXPROT, |
---|
427 | ("%s(%u): protocol too big for %s", __func__, proto, name)); |
---|
428 | |
---|
429 | NETISR_WLOCK(); |
---|
430 | KASSERT(np[proto].np_handler != NULL, |
---|
431 | ("%s(%u): protocol not registered for %s", __func__, proto, |
---|
432 | name)); |
---|
433 | |
---|
434 | for (i = 0; i <= mp_maxid; i++) { |
---|
435 | if (CPU_ABSENT(i)) |
---|
436 | continue; |
---|
437 | npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; |
---|
438 | npwp->nw_qdrops = 0; |
---|
439 | } |
---|
440 | NETISR_WUNLOCK(); |
---|
441 | } |
---|
442 | |
---|
443 | /* |
---|
444 | * Query the current drop counters across all workstreams for a protocol. |
---|
445 | */ |
---|
446 | void |
---|
447 | netisr_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp) |
---|
448 | { |
---|
449 | struct netisr_work *npwp; |
---|
450 | struct rm_priotracker tracker; |
---|
451 | #ifdef INVARIANTS |
---|
452 | const char *name; |
---|
453 | #endif |
---|
454 | u_int i, proto; |
---|
455 | |
---|
456 | *qdropp = 0; |
---|
457 | proto = nhp->nh_proto; |
---|
458 | #ifdef INVARIANTS |
---|
459 | name = nhp->nh_name; |
---|
460 | #endif |
---|
461 | KASSERT(proto < NETISR_MAXPROT, |
---|
462 | ("%s(%u): protocol too big for %s", __func__, proto, name)); |
---|
463 | |
---|
464 | NETISR_RLOCK(&tracker); |
---|
465 | KASSERT(np[proto].np_handler != NULL, |
---|
466 | ("%s(%u): protocol not registered for %s", __func__, proto, |
---|
467 | name)); |
---|
468 | |
---|
469 | for (i = 0; i <= mp_maxid; i++) { |
---|
470 | if (CPU_ABSENT(i)) |
---|
471 | continue; |
---|
472 | npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; |
---|
473 | *qdropp += npwp->nw_qdrops; |
---|
474 | } |
---|
475 | NETISR_RUNLOCK(&tracker); |
---|
476 | } |
---|
477 | |
---|
478 | /* |
---|
479 | * Query the current queue limit for per-workstream queues for a protocol. |
---|
480 | */ |
---|
481 | void |
---|
482 | netisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp) |
---|
483 | { |
---|
484 | struct rm_priotracker tracker; |
---|
485 | #ifdef INVARIANTS |
---|
486 | const char *name; |
---|
487 | #endif |
---|
488 | u_int proto; |
---|
489 | |
---|
490 | proto = nhp->nh_proto; |
---|
491 | #ifdef INVARIANTS |
---|
492 | name = nhp->nh_name; |
---|
493 | #endif |
---|
494 | KASSERT(proto < NETISR_MAXPROT, |
---|
495 | ("%s(%u): protocol too big for %s", __func__, proto, name)); |
---|
496 | |
---|
497 | NETISR_RLOCK(&tracker); |
---|
498 | KASSERT(np[proto].np_handler != NULL, |
---|
499 | ("%s(%u): protocol not registered for %s", __func__, proto, |
---|
500 | name)); |
---|
501 | *qlimitp = np[proto].np_qlimit; |
---|
502 | NETISR_RUNLOCK(&tracker); |
---|
503 | } |
---|
504 | |
---|
505 | /* |
---|
506 | * Update the queue limit across per-workstream queues for a protocol. We |
---|
507 | * simply change the limits, and don't drain overflowed packets as they will |
---|
508 | * (hopefully) take care of themselves shortly. |
---|
509 | */ |
---|
510 | int |
---|
511 | netisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit) |
---|
512 | { |
---|
513 | struct netisr_work *npwp; |
---|
514 | #ifdef INVARIANTS |
---|
515 | const char *name; |
---|
516 | #endif |
---|
517 | u_int i, proto; |
---|
518 | |
---|
519 | if (qlimit > netisr_maxqlimit) |
---|
520 | return (EINVAL); |
---|
521 | |
---|
522 | proto = nhp->nh_proto; |
---|
523 | #ifdef INVARIANTS |
---|
524 | name = nhp->nh_name; |
---|
525 | #endif |
---|
526 | KASSERT(proto < NETISR_MAXPROT, |
---|
527 | ("%s(%u): protocol too big for %s", __func__, proto, name)); |
---|
528 | |
---|
529 | NETISR_WLOCK(); |
---|
530 | KASSERT(np[proto].np_handler != NULL, |
---|
531 | ("%s(%u): protocol not registered for %s", __func__, proto, |
---|
532 | name)); |
---|
533 | |
---|
534 | np[proto].np_qlimit = qlimit; |
---|
535 | for (i = 0; i <= mp_maxid; i++) { |
---|
536 | if (CPU_ABSENT(i)) |
---|
537 | continue; |
---|
538 | npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; |
---|
539 | npwp->nw_qlimit = qlimit; |
---|
540 | } |
---|
541 | NETISR_WUNLOCK(); |
---|
542 | return (0); |
---|
543 | } |
---|
544 | |
---|
545 | #ifndef __rtems__ |
---|
546 | /* |
---|
547 | * Drain all packets currently held in a particular protocol work queue. |
---|
548 | */ |
---|
549 | static void |
---|
550 | netisr_drain_proto(struct netisr_work *npwp) |
---|
551 | { |
---|
552 | struct mbuf *m; |
---|
553 | |
---|
554 | /* |
---|
555 | * We would assert the lock on the workstream but it's not passed in. |
---|
556 | */ |
---|
557 | while ((m = npwp->nw_head) != NULL) { |
---|
558 | npwp->nw_head = m->m_nextpkt; |
---|
559 | m->m_nextpkt = NULL; |
---|
560 | if (npwp->nw_head == NULL) |
---|
561 | npwp->nw_tail = NULL; |
---|
562 | npwp->nw_len--; |
---|
563 | m_freem(m); |
---|
564 | } |
---|
565 | KASSERT(npwp->nw_tail == NULL, ("%s: tail", __func__)); |
---|
566 | KASSERT(npwp->nw_len == 0, ("%s: len", __func__)); |
---|
567 | } |
---|
568 | |
---|
569 | /* |
---|
570 | * Remove the registration of a network protocol, which requires clearing |
---|
571 | * per-protocol fields across all workstreams, including freeing all mbufs in |
---|
572 | * the queues at time of unregister. All work in netisr is briefly suspended |
---|
573 | * while this takes place. |
---|
574 | */ |
---|
575 | void |
---|
576 | netisr_unregister(const struct netisr_handler *nhp) |
---|
577 | { |
---|
578 | struct netisr_work *npwp; |
---|
579 | #ifdef INVARIANTS |
---|
580 | const char *name; |
---|
581 | #endif |
---|
582 | u_int i, proto; |
---|
583 | |
---|
584 | proto = nhp->nh_proto; |
---|
585 | #ifdef INVARIANTS |
---|
586 | name = nhp->nh_name; |
---|
587 | #endif |
---|
588 | KASSERT(proto < NETISR_MAXPROT, |
---|
589 | ("%s(%u): protocol too big for %s", __func__, proto, name)); |
---|
590 | |
---|
591 | NETISR_WLOCK(); |
---|
592 | KASSERT(np[proto].np_handler != NULL, |
---|
593 | ("%s(%u): protocol not registered for %s", __func__, proto, |
---|
594 | name)); |
---|
595 | |
---|
596 | np[proto].np_name = NULL; |
---|
597 | np[proto].np_handler = NULL; |
---|
598 | np[proto].np_m2flow = NULL; |
---|
599 | np[proto].np_m2cpuid = NULL; |
---|
600 | np[proto].np_qlimit = 0; |
---|
601 | np[proto].np_policy = 0; |
---|
602 | for (i = 0; i <= mp_maxid; i++) { |
---|
603 | if (CPU_ABSENT(i)) |
---|
604 | continue; |
---|
605 | npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto]; |
---|
606 | netisr_drain_proto(npwp); |
---|
607 | bzero(npwp, sizeof(*npwp)); |
---|
608 | } |
---|
609 | NETISR_WUNLOCK(); |
---|
610 | } |
---|
611 | #endif /* __rtems__ */ |
---|
612 | |
---|
613 | /* |
---|
614 | * Look up the workstream given a packet and source identifier. Do this by |
---|
615 | * checking the protocol's policy, and optionally call out to the protocol |
---|
616 | * for assistance if required. |
---|
617 | */ |
---|
618 | static struct mbuf * |
---|
619 | netisr_select_cpuid(struct netisr_proto *npp, uintptr_t source, |
---|
620 | struct mbuf *m, u_int *cpuidp) |
---|
621 | { |
---|
622 | struct ifnet *ifp; |
---|
623 | |
---|
624 | NETISR_LOCK_ASSERT(); |
---|
625 | |
---|
626 | /* |
---|
627 | * In the event we have only one worker, shortcut and deliver to it |
---|
628 | * without further ado. |
---|
629 | */ |
---|
630 | if (nws_count == 1) { |
---|
631 | *cpuidp = nws_array[0]; |
---|
632 | return (m); |
---|
633 | } |
---|
634 | |
---|
635 | /* |
---|
636 | * What happens next depends on the policy selected by the protocol. |
---|
637 | * If we want to support per-interface policies, we should do that |
---|
638 | * here first. |
---|
639 | */ |
---|
640 | switch (npp->np_policy) { |
---|
641 | case NETISR_POLICY_CPU: |
---|
642 | return (npp->np_m2cpuid(m, source, cpuidp)); |
---|
643 | |
---|
644 | case NETISR_POLICY_FLOW: |
---|
645 | if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) { |
---|
646 | m = npp->np_m2flow(m, source); |
---|
647 | if (m == NULL) |
---|
648 | return (NULL); |
---|
649 | } |
---|
650 | if (m->m_flags & M_FLOWID) { |
---|
651 | *cpuidp = |
---|
652 | netisr_default_flow2cpu(m->m_pkthdr.flowid); |
---|
653 | return (m); |
---|
654 | } |
---|
655 | /* FALLTHROUGH */ |
---|
656 | |
---|
657 | case NETISR_POLICY_SOURCE: |
---|
658 | ifp = m->m_pkthdr.rcvif; |
---|
659 | if (ifp != NULL) |
---|
660 | *cpuidp = nws_array[(ifp->if_index + source) % |
---|
661 | nws_count]; |
---|
662 | else |
---|
663 | *cpuidp = nws_array[source % nws_count]; |
---|
664 | return (m); |
---|
665 | |
---|
666 | default: |
---|
667 | panic("%s: invalid policy %u for %s", __func__, |
---|
668 | npp->np_policy, npp->np_name); |
---|
669 | } |
---|
670 | } |
---|
671 | |
---|
672 | /* |
---|
673 | * Process packets associated with a workstream and protocol. For reasons of |
---|
674 | * fairness, we process up to one complete netisr queue at a time, moving the |
---|
675 | * queue to a stack-local queue for processing, but do not loop refreshing |
---|
676 | * from the global queue. The caller is responsible for deciding whether to |
---|
677 | * loop, and for setting the NWS_RUNNING flag. The passed workstream will be |
---|
678 | * locked on entry and relocked before return, but will be released while |
---|
679 | * processing. The number of packets processed is returned. |
---|
680 | */ |
---|
681 | static u_int |
---|
682 | netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto) |
---|
683 | { |
---|
684 | struct netisr_work local_npw, *npwp; |
---|
685 | u_int handled; |
---|
686 | struct mbuf *m; |
---|
687 | |
---|
688 | NETISR_LOCK_ASSERT(); |
---|
689 | NWS_LOCK_ASSERT(nwsp); |
---|
690 | |
---|
691 | KASSERT(nwsp->nws_flags & NWS_RUNNING, |
---|
692 | ("%s(%u): not running", __func__, proto)); |
---|
693 | KASSERT(proto >= 0 && proto < NETISR_MAXPROT, |
---|
694 | ("%s(%u): invalid proto\n", __func__, proto)); |
---|
695 | |
---|
696 | npwp = &nwsp->nws_work[proto]; |
---|
697 | if (npwp->nw_len == 0) |
---|
698 | return (0); |
---|
699 | |
---|
700 | /* |
---|
701 | * Move the global work queue to a thread-local work queue. |
---|
702 | * |
---|
703 | * Notice that this means the effective maximum length of the queue |
---|
704 | * is actually twice that of the maximum queue length specified in |
---|
705 | * the protocol registration call. |
---|
706 | */ |
---|
707 | handled = npwp->nw_len; |
---|
708 | local_npw = *npwp; |
---|
709 | npwp->nw_head = NULL; |
---|
710 | npwp->nw_tail = NULL; |
---|
711 | npwp->nw_len = 0; |
---|
712 | nwsp->nws_pendingbits &= ~(1 << proto); |
---|
713 | NWS_UNLOCK(nwsp); |
---|
714 | while ((m = local_npw.nw_head) != NULL) { |
---|
715 | local_npw.nw_head = m->m_nextpkt; |
---|
716 | m->m_nextpkt = NULL; |
---|
717 | if (local_npw.nw_head == NULL) |
---|
718 | local_npw.nw_tail = NULL; |
---|
719 | local_npw.nw_len--; |
---|
720 | VNET_ASSERT(m->m_pkthdr.rcvif != NULL); |
---|
721 | CURVNET_SET(m->m_pkthdr.rcvif->if_vnet); |
---|
722 | np[proto].np_handler(m); |
---|
723 | CURVNET_RESTORE(); |
---|
724 | } |
---|
725 | KASSERT(local_npw.nw_len == 0, |
---|
726 | ("%s(%u): len %u", __func__, proto, local_npw.nw_len)); |
---|
727 | if (np[proto].np_drainedcpu) |
---|
728 | np[proto].np_drainedcpu(nwsp->nws_cpu); |
---|
729 | NWS_LOCK(nwsp); |
---|
730 | npwp->nw_handled += handled; |
---|
731 | return (handled); |
---|
732 | } |
---|
733 | |
---|
734 | /* |
---|
735 | * SWI handler for netisr -- processes prackets in a set of workstreams that |
---|
736 | * it owns, woken up by calls to NWS_SIGNAL(). If this workstream is already |
---|
737 | * being direct dispatched, go back to sleep and wait for the dispatching |
---|
738 | * thread to wake us up again. |
---|
739 | */ |
---|
740 | static void |
---|
741 | swi_net(void *arg) |
---|
742 | { |
---|
743 | #ifdef NETISR_LOCKING |
---|
744 | struct rm_priotracker tracker; |
---|
745 | #endif |
---|
746 | struct netisr_workstream *nwsp; |
---|
747 | u_int bits, prot; |
---|
748 | |
---|
749 | nwsp = arg; |
---|
750 | |
---|
751 | #ifdef DEVICE_POLLING |
---|
752 | KASSERT(nws_count == 1, |
---|
753 | ("%s: device_polling but nws_count != 1", __func__)); |
---|
754 | netisr_poll(); |
---|
755 | #endif |
---|
756 | #ifdef NETISR_LOCKING |
---|
757 | NETISR_RLOCK(&tracker); |
---|
758 | #endif |
---|
759 | NWS_LOCK(nwsp); |
---|
760 | KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running")); |
---|
761 | if (nwsp->nws_flags & NWS_DISPATCHING) |
---|
762 | goto out; |
---|
763 | nwsp->nws_flags |= NWS_RUNNING; |
---|
764 | nwsp->nws_flags &= ~NWS_SCHEDULED; |
---|
765 | while ((bits = nwsp->nws_pendingbits) != 0) { |
---|
766 | while ((prot = ffs(bits)) != 0) { |
---|
767 | prot--; |
---|
768 | bits &= ~(1 << prot); |
---|
769 | (void)netisr_process_workstream_proto(nwsp, prot); |
---|
770 | } |
---|
771 | } |
---|
772 | nwsp->nws_flags &= ~NWS_RUNNING; |
---|
773 | out: |
---|
774 | NWS_UNLOCK(nwsp); |
---|
775 | #ifdef NETISR_LOCKING |
---|
776 | NETISR_RUNLOCK(&tracker); |
---|
777 | #endif |
---|
778 | #ifdef DEVICE_POLLING |
---|
779 | netisr_pollmore(); |
---|
780 | #endif |
---|
781 | } |
---|
782 | |
---|
783 | static int |
---|
784 | netisr_queue_workstream(struct netisr_workstream *nwsp, u_int proto, |
---|
785 | struct netisr_work *npwp, struct mbuf *m, int *dosignalp) |
---|
786 | { |
---|
787 | |
---|
788 | NWS_LOCK_ASSERT(nwsp); |
---|
789 | |
---|
790 | *dosignalp = 0; |
---|
791 | if (npwp->nw_len < npwp->nw_qlimit) { |
---|
792 | m->m_nextpkt = NULL; |
---|
793 | if (npwp->nw_head == NULL) { |
---|
794 | npwp->nw_head = m; |
---|
795 | npwp->nw_tail = m; |
---|
796 | } else { |
---|
797 | npwp->nw_tail->m_nextpkt = m; |
---|
798 | npwp->nw_tail = m; |
---|
799 | } |
---|
800 | npwp->nw_len++; |
---|
801 | if (npwp->nw_len > npwp->nw_watermark) |
---|
802 | npwp->nw_watermark = npwp->nw_len; |
---|
803 | nwsp->nws_pendingbits |= (1 << proto); |
---|
804 | if (!(nwsp->nws_flags & |
---|
805 | (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED))) { |
---|
806 | nwsp->nws_flags |= NWS_SCHEDULED; |
---|
807 | *dosignalp = 1; /* Defer until unlocked. */ |
---|
808 | } |
---|
809 | npwp->nw_queued++; |
---|
810 | return (0); |
---|
811 | } else { |
---|
812 | m_freem(m); |
---|
813 | npwp->nw_qdrops++; |
---|
814 | return (ENOBUFS); |
---|
815 | } |
---|
816 | } |
---|
817 | |
---|
818 | static int |
---|
819 | netisr_queue_internal(u_int proto, struct mbuf *m, u_int cpuid) |
---|
820 | { |
---|
821 | struct netisr_workstream *nwsp; |
---|
822 | struct netisr_work *npwp; |
---|
823 | int dosignal, error; |
---|
824 | |
---|
825 | #ifdef NETISR_LOCKING |
---|
826 | NETISR_LOCK_ASSERT(); |
---|
827 | #endif |
---|
828 | KASSERT(cpuid <= mp_maxid, ("%s: cpuid too big (%u, %u)", __func__, |
---|
829 | cpuid, mp_maxid)); |
---|
830 | KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); |
---|
831 | |
---|
832 | dosignal = 0; |
---|
833 | error = 0; |
---|
834 | nwsp = DPCPU_ID_PTR(cpuid, nws); |
---|
835 | npwp = &nwsp->nws_work[proto]; |
---|
836 | NWS_LOCK(nwsp); |
---|
837 | error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal); |
---|
838 | NWS_UNLOCK(nwsp); |
---|
839 | if (dosignal) |
---|
840 | NWS_SIGNAL(nwsp); |
---|
841 | return (error); |
---|
842 | } |
---|
843 | |
---|
844 | int |
---|
845 | netisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m) |
---|
846 | { |
---|
847 | #ifdef NETISR_LOCKING |
---|
848 | struct rm_priotracker tracker; |
---|
849 | #endif |
---|
850 | u_int cpuid; |
---|
851 | int error; |
---|
852 | |
---|
853 | KASSERT(proto < NETISR_MAXPROT, |
---|
854 | ("%s: invalid proto %u", __func__, proto)); |
---|
855 | |
---|
856 | #ifdef NETISR_LOCKING |
---|
857 | NETISR_RLOCK(&tracker); |
---|
858 | #endif |
---|
859 | KASSERT(np[proto].np_handler != NULL, |
---|
860 | ("%s: invalid proto %u", __func__, proto)); |
---|
861 | |
---|
862 | m = netisr_select_cpuid(&np[proto], source, m, &cpuid); |
---|
863 | if (m != NULL) { |
---|
864 | KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, |
---|
865 | cpuid)); |
---|
866 | error = netisr_queue_internal(proto, m, cpuid); |
---|
867 | } else |
---|
868 | error = ENOBUFS; |
---|
869 | #ifdef NETISR_LOCKING |
---|
870 | NETISR_RUNLOCK(&tracker); |
---|
871 | #endif |
---|
872 | return (error); |
---|
873 | } |
---|
874 | |
---|
875 | int |
---|
876 | netisr_queue(u_int proto, struct mbuf *m) |
---|
877 | { |
---|
878 | |
---|
879 | return (netisr_queue_src(proto, 0, m)); |
---|
880 | } |
---|
881 | |
---|
882 | /* |
---|
883 | * Dispatch a packet for netisr processing, direct dispatch permitted by |
---|
884 | * calling context. |
---|
885 | */ |
---|
886 | int |
---|
887 | netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) |
---|
888 | { |
---|
889 | #ifdef NETISR_LOCKING |
---|
890 | struct rm_priotracker tracker; |
---|
891 | #endif |
---|
892 | struct netisr_workstream *nwsp; |
---|
893 | struct netisr_work *npwp; |
---|
894 | int dosignal, error; |
---|
895 | u_int cpuid; |
---|
896 | |
---|
897 | /* |
---|
898 | * If direct dispatch is entirely disabled, fall back on queueing. |
---|
899 | */ |
---|
900 | if (!netisr_direct) |
---|
901 | return (netisr_queue_src(proto, source, m)); |
---|
902 | |
---|
903 | KASSERT(proto < NETISR_MAXPROT, |
---|
904 | ("%s: invalid proto %u", __func__, proto)); |
---|
905 | #ifdef NETISR_LOCKING |
---|
906 | NETISR_RLOCK(&tracker); |
---|
907 | #endif |
---|
908 | KASSERT(np[proto].np_handler != NULL, |
---|
909 | ("%s: invalid proto %u", __func__, proto)); |
---|
910 | |
---|
911 | /* |
---|
912 | * If direct dispatch is forced, then unconditionally dispatch |
---|
913 | * without a formal CPU selection. Borrow the current CPU's stats, |
---|
914 | * even if there's no worker on it. In this case we don't update |
---|
915 | * nws_flags because all netisr processing will be source ordered due |
---|
916 | * to always being forced to directly dispatch. |
---|
917 | */ |
---|
918 | if (netisr_direct_force) { |
---|
919 | nwsp = DPCPU_PTR(nws); |
---|
920 | npwp = &nwsp->nws_work[proto]; |
---|
921 | npwp->nw_dispatched++; |
---|
922 | npwp->nw_handled++; |
---|
923 | np[proto].np_handler(m); |
---|
924 | error = 0; |
---|
925 | goto out_unlock; |
---|
926 | } |
---|
927 | |
---|
928 | /* |
---|
929 | * Otherwise, we execute in a hybrid mode where we will try to direct |
---|
930 | * dispatch if we're on the right CPU and the netisr worker isn't |
---|
931 | * already running. |
---|
932 | */ |
---|
933 | m = netisr_select_cpuid(&np[proto], source, m, &cpuid); |
---|
934 | if (m == NULL) { |
---|
935 | error = ENOBUFS; |
---|
936 | goto out_unlock; |
---|
937 | } |
---|
938 | KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); |
---|
939 | #ifndef __rtems__ |
---|
940 | sched_pin(); |
---|
941 | #endif /* __rtems__ */ |
---|
942 | if (cpuid != curcpu) |
---|
943 | goto queue_fallback; |
---|
944 | nwsp = DPCPU_PTR(nws); |
---|
945 | npwp = &nwsp->nws_work[proto]; |
---|
946 | |
---|
947 | /*- |
---|
948 | * We are willing to direct dispatch only if three conditions hold: |
---|
949 | * |
---|
950 | * (1) The netisr worker isn't already running, |
---|
951 | * (2) Another thread isn't already directly dispatching, and |
---|
952 | * (3) The netisr hasn't already been woken up. |
---|
953 | */ |
---|
954 | NWS_LOCK(nwsp); |
---|
955 | if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) { |
---|
956 | error = netisr_queue_workstream(nwsp, proto, npwp, m, |
---|
957 | &dosignal); |
---|
958 | NWS_UNLOCK(nwsp); |
---|
959 | if (dosignal) |
---|
960 | NWS_SIGNAL(nwsp); |
---|
961 | goto out_unpin; |
---|
962 | } |
---|
963 | |
---|
964 | /* |
---|
965 | * The current thread is now effectively the netisr worker, so set |
---|
966 | * the dispatching flag to prevent concurrent processing of the |
---|
967 | * stream from another thread (even the netisr worker), which could |
---|
968 | * otherwise lead to effective misordering of the stream. |
---|
969 | */ |
---|
970 | nwsp->nws_flags |= NWS_DISPATCHING; |
---|
971 | NWS_UNLOCK(nwsp); |
---|
972 | np[proto].np_handler(m); |
---|
973 | NWS_LOCK(nwsp); |
---|
974 | nwsp->nws_flags &= ~NWS_DISPATCHING; |
---|
975 | npwp->nw_handled++; |
---|
976 | npwp->nw_hybrid_dispatched++; |
---|
977 | |
---|
978 | /* |
---|
979 | * If other work was enqueued by another thread while we were direct |
---|
980 | * dispatching, we need to signal the netisr worker to do that work. |
---|
981 | * In the future, we might want to do some of that work in the |
---|
982 | * current thread, rather than trigger further context switches. If |
---|
983 | * so, we'll want to establish a reasonable bound on the work done in |
---|
984 | * the "borrowed" context. |
---|
985 | */ |
---|
986 | if (nwsp->nws_pendingbits != 0) { |
---|
987 | nwsp->nws_flags |= NWS_SCHEDULED; |
---|
988 | dosignal = 1; |
---|
989 | } else |
---|
990 | dosignal = 0; |
---|
991 | NWS_UNLOCK(nwsp); |
---|
992 | if (dosignal) |
---|
993 | NWS_SIGNAL(nwsp); |
---|
994 | error = 0; |
---|
995 | goto out_unpin; |
---|
996 | |
---|
997 | queue_fallback: |
---|
998 | error = netisr_queue_internal(proto, m, cpuid); |
---|
999 | out_unpin: |
---|
1000 | #ifndef __rtems__ |
---|
1001 | sched_unpin(); |
---|
1002 | #endif /* __rtems__ */ |
---|
1003 | out_unlock: |
---|
1004 | #ifdef NETISR_LOCKING |
---|
1005 | NETISR_RUNLOCK(&tracker); |
---|
1006 | #endif |
---|
1007 | return (error); |
---|
1008 | } |
---|
1009 | |
---|
1010 | int |
---|
1011 | netisr_dispatch(u_int proto, struct mbuf *m) |
---|
1012 | { |
---|
1013 | |
---|
1014 | return (netisr_dispatch_src(proto, 0, m)); |
---|
1015 | } |
---|
1016 | |
---|
1017 | #ifdef DEVICE_POLLING |
---|
1018 | /* |
---|
1019 | * Kernel polling borrows a netisr thread to run interface polling in; this |
---|
1020 | * function allows kernel polling to request that the netisr thread be |
---|
1021 | * scheduled even if no packets are pending for protocols. |
---|
1022 | */ |
---|
1023 | void |
---|
1024 | netisr_sched_poll(void) |
---|
1025 | { |
---|
1026 | struct netisr_workstream *nwsp; |
---|
1027 | |
---|
1028 | nwsp = DPCPU_ID_PTR(nws_array[0], nws); |
---|
1029 | NWS_SIGNAL(nwsp); |
---|
1030 | } |
---|
1031 | #endif |
---|
1032 | |
---|
1033 | static void |
---|
1034 | netisr_start_swi(u_int cpuid, struct pcpu *pc) |
---|
1035 | { |
---|
1036 | char swiname[12]; |
---|
1037 | struct netisr_workstream *nwsp; |
---|
1038 | int error; |
---|
1039 | |
---|
1040 | KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid)); |
---|
1041 | |
---|
1042 | nwsp = DPCPU_ID_PTR(cpuid, nws); |
---|
1043 | mtx_init(&nwsp->nws_mtx, "netisr_mtx", NULL, MTX_DEF); |
---|
1044 | nwsp->nws_cpu = cpuid; |
---|
1045 | snprintf(swiname, sizeof(swiname), "netisr %u", cpuid); |
---|
1046 | error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp, |
---|
1047 | SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie); |
---|
1048 | if (error) |
---|
1049 | panic("%s: swi_add %d", __func__, error); |
---|
1050 | pc->pc_netisr = nwsp->nws_intr_event; |
---|
1051 | #ifndef __rtems__ |
---|
1052 | if (netisr_bindthreads) { |
---|
1053 | error = intr_event_bind(nwsp->nws_intr_event, cpuid); |
---|
1054 | if (error != 0) |
---|
1055 | printf("%s: cpu %u: intr_event_bind: %d", __func__, |
---|
1056 | cpuid, error); |
---|
1057 | } |
---|
1058 | #endif |
---|
1059 | NETISR_WLOCK(); |
---|
1060 | nws_array[nws_count] = nwsp->nws_cpu; |
---|
1061 | nws_count++; |
---|
1062 | NETISR_WUNLOCK(); |
---|
1063 | } |
---|
1064 | |
---|
1065 | /* |
---|
1066 | * Initialize the netisr subsystem. We rely on BSS and static initialization |
---|
1067 | * of most fields in global data structures. |
---|
1068 | * |
---|
1069 | * Start a worker thread for the boot CPU so that we can support network |
---|
1070 | * traffic immediately in case the network stack is used before additional |
---|
1071 | * CPUs are started (for example, diskless boot). |
---|
1072 | */ |
---|
1073 | static void |
---|
1074 | netisr_init(void *arg) |
---|
1075 | { |
---|
1076 | |
---|
1077 | KASSERT(curcpu == 0, ("%s: not on CPU 0", __func__)); |
---|
1078 | |
---|
1079 | NETISR_LOCK_INIT(); |
---|
1080 | if (netisr_maxthreads < 1) |
---|
1081 | netisr_maxthreads = 1; |
---|
1082 | if (netisr_maxthreads > mp_ncpus) { |
---|
1083 | printf("netisr_init: forcing maxthreads from %d to %d\n", |
---|
1084 | netisr_maxthreads, mp_ncpus); |
---|
1085 | netisr_maxthreads = mp_ncpus; |
---|
1086 | } |
---|
1087 | if (netisr_defaultqlimit > netisr_maxqlimit) { |
---|
1088 | printf("netisr_init: forcing defaultqlimit from %d to %d\n", |
---|
1089 | netisr_defaultqlimit, netisr_maxqlimit); |
---|
1090 | netisr_defaultqlimit = netisr_maxqlimit; |
---|
1091 | } |
---|
1092 | #ifdef DEVICE_POLLING |
---|
1093 | /* |
---|
1094 | * The device polling code is not yet aware of how to deal with |
---|
1095 | * multiple netisr threads, so for the time being compiling in device |
---|
1096 | * polling disables parallel netisr workers. |
---|
1097 | */ |
---|
1098 | if (netisr_maxthreads != 1 || netisr_bindthreads != 0) { |
---|
1099 | printf("netisr_init: forcing maxthreads to 1 and " |
---|
1100 | "bindthreads to 0 for device polling\n"); |
---|
1101 | netisr_maxthreads = 1; |
---|
1102 | netisr_bindthreads = 0; |
---|
1103 | } |
---|
1104 | #endif |
---|
1105 | |
---|
1106 | netisr_start_swi(curcpu, pcpu_find(curcpu)); |
---|
1107 | } |
---|
1108 | SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL); |
---|
1109 | |
---|
1110 | /* |
---|
1111 | * Start worker threads for additional CPUs. No attempt to gracefully handle |
---|
1112 | * work reassignment, we don't yet support dynamic reconfiguration. |
---|
1113 | */ |
---|
1114 | static void |
---|
1115 | netisr_start(void *arg) |
---|
1116 | { |
---|
1117 | struct pcpu *pc; |
---|
1118 | |
---|
1119 | SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { |
---|
1120 | if (nws_count >= netisr_maxthreads) |
---|
1121 | break; |
---|
1122 | /* XXXRW: Is skipping absent CPUs still required here? */ |
---|
1123 | if (CPU_ABSENT(pc->pc_cpuid)) |
---|
1124 | continue; |
---|
1125 | /* Worker will already be present for boot CPU. */ |
---|
1126 | if (pc->pc_netisr != NULL) |
---|
1127 | continue; |
---|
1128 | netisr_start_swi(pc->pc_cpuid, pc); |
---|
1129 | } |
---|
1130 | } |
---|
1131 | SYSINIT(netisr_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr_start, NULL); |
---|
1132 | |
---|
1133 | #ifdef DDB |
---|
1134 | DB_SHOW_COMMAND(netisr, db_show_netisr) |
---|
1135 | { |
---|
1136 | struct netisr_workstream *nwsp; |
---|
1137 | struct netisr_work *nwp; |
---|
1138 | int first, proto; |
---|
1139 | u_int cpuid; |
---|
1140 | |
---|
1141 | db_printf("%3s %6s %5s %5s %5s %8s %8s %8s %8s\n", "CPU", "Proto", |
---|
1142 | "Len", "WMark", "Max", "Disp", "HDisp", "Drop", "Queue"); |
---|
1143 | for (cpuid = 0; cpuid <= mp_maxid; cpuid++) { |
---|
1144 | if (CPU_ABSENT(cpuid)) |
---|
1145 | continue; |
---|
1146 | nwsp = DPCPU_ID_PTR(cpuid, nws); |
---|
1147 | if (nwsp->nws_intr_event == NULL) |
---|
1148 | continue; |
---|
1149 | first = 1; |
---|
1150 | for (proto = 0; proto < NETISR_MAXPROT; proto++) { |
---|
1151 | if (np[proto].np_handler == NULL) |
---|
1152 | continue; |
---|
1153 | nwp = &nwsp->nws_work[proto]; |
---|
1154 | if (first) { |
---|
1155 | db_printf("%3d ", cpuid); |
---|
1156 | first = 0; |
---|
1157 | } else |
---|
1158 | db_printf("%3s ", ""); |
---|
1159 | db_printf( |
---|
1160 | "%6s %5d %5d %5d %8ju %8ju %8ju %8ju\n", |
---|
1161 | np[proto].np_name, nwp->nw_len, |
---|
1162 | nwp->nw_watermark, nwp->nw_qlimit, |
---|
1163 | nwp->nw_dispatched, nwp->nw_hybrid_dispatched, |
---|
1164 | nwp->nw_qdrops, nwp->nw_queued); |
---|
1165 | } |
---|
1166 | } |
---|
1167 | } |
---|
1168 | #endif |
---|