source: rtems-libbsd/freebsd/sys/net/netisr.c @ 760e894

4.1155-freebsd-126-freebsd-12freebsd-9.3
Last change on this file since 760e894 was 760e894, checked in by Sebastian Huber <sebastian.huber@…>, on Nov 12, 2013 at 8:36:18 AM

NETISR(9): Fix NULL pointer access

  • Property mode set to 100644
File size: 42.8 KB
Line 
1#include <machine/rtems-bsd-kernel-space.h>
2
3/*-
4 * Copyright (c) 2007-2009 Robert N. M. Watson
5 * Copyright (c) 2010-2011 Juniper Networks, Inc.
6 * All rights reserved.
7 *
8 * This software was developed by Robert N. M. Watson under contract
9 * to Juniper Networks, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 *    notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 *    notice, this list of conditions and the following disclaimer in the
18 *    documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD$");
35
36/*
37 * netisr is a packet dispatch service, allowing synchronous (directly
38 * dispatched) and asynchronous (deferred dispatch) processing of packets by
39 * registered protocol handlers.  Callers pass a protocol identifier and
40 * packet to netisr, along with a direct dispatch hint, and work will either
41 * be immediately processed by the registered handler, or passed to a
42 * software interrupt (SWI) thread for deferred dispatch.  Callers will
43 * generally select one or the other based on:
44 *
45 * - Whether directly dispatching a netisr handler lead to code reentrance or
46 *   lock recursion, such as entering the socket code from the socket code.
47 * - Whether directly dispatching a netisr handler lead to recursive
48 *   processing, such as when decapsulating several wrapped layers of tunnel
49 *   information (IPSEC within IPSEC within ...).
50 *
51 * Maintaining ordering for protocol streams is a critical design concern.
52 * Enforcing ordering limits the opportunity for concurrency, but maintains
53 * the strong ordering requirements found in some protocols, such as TCP.  Of
54 * related concern is CPU affinity--it is desirable to process all data
55 * associated with a particular stream on the same CPU over time in order to
56 * avoid acquiring locks associated with the connection on different CPUs,
57 * keep connection data in one cache, and to generally encourage associated
58 * user threads to live on the same CPU as the stream.  It's also desirable
59 * to avoid lock migration and contention where locks are associated with
60 * more than one flow.
61 *
62 * netisr supports several policy variations, represented by the
63 * NETISR_POLICY_* constants, allowing protocols to play various roles in
64 * identifying flows, assigning work to CPUs, etc.  These are described in
65 * netisr.h.
66 */
67
68#include <rtems/bsd/local/opt_ddb.h>
69#include <rtems/bsd/local/opt_device_polling.h>
70
71#include <rtems/bsd/sys/param.h>
72#include <sys/bus.h>
73#include <sys/kernel.h>
74#include <sys/kthread.h>
75#include <sys/interrupt.h>
76#include <rtems/bsd/sys/lock.h>
77#include <sys/mbuf.h>
78#include <sys/mutex.h>
79#include <sys/pcpu.h>
80#include <sys/proc.h>
81#include <sys/rmlock.h>
82#include <sys/sched.h>
83#include <sys/smp.h>
84#include <sys/socket.h>
85#include <sys/sysctl.h>
86#include <sys/systm.h>
87
88#ifdef DDB
89#include <ddb/ddb.h>
90#endif
91
92#define _WANT_NETISR_INTERNAL   /* Enable definitions from netisr_internal.h */
93#include <net/if.h>
94#include <net/if_var.h>
95#include <net/netisr.h>
96#include <net/netisr_internal.h>
97#include <net/vnet.h>
98
99/*-
100 * Synchronize use and modification of the registered netisr data structures;
101 * acquire a read lock while modifying the set of registered protocols to
102 * prevent partially registered or unregistered protocols from being run.
103 *
104 * The following data structures and fields are protected by this lock:
105 *
106 * - The netisr_proto array, including all fields of struct netisr_proto.
107 * - The nws array, including all fields of struct netisr_worker.
108 * - The nws_array array.
109 *
110 * Note: the NETISR_LOCKING define controls whether read locks are acquired
111 * in packet processing paths requiring netisr registration stability.  This
112 * is disabled by default as it can lead to measurable performance
113 * degradation even with rmlocks (3%-6% for loopback ping-pong traffic), and
114 * because netisr registration and unregistration is extremely rare at
115 * runtime.  If it becomes more common, this decision should be revisited.
116 *
117 * XXXRW: rmlocks don't support assertions.
118 */
119static struct rmlock    netisr_rmlock;
120#define NETISR_LOCK_INIT()      rm_init_flags(&netisr_rmlock, "netisr", \
121                                    RM_NOWITNESS)
122#define NETISR_LOCK_ASSERT()
123#define NETISR_RLOCK(tracker)   rm_rlock(&netisr_rmlock, (tracker))
124#define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker))
125#define NETISR_WLOCK()          rm_wlock(&netisr_rmlock)
126#define NETISR_WUNLOCK()        rm_wunlock(&netisr_rmlock)
127/* #define      NETISR_LOCKING */
128
129static SYSCTL_NODE(_net, OID_AUTO, isr, CTLFLAG_RW, 0, "netisr");
130
131/*-
132 * Three global direct dispatch policies are supported:
133 *
134 * NETISR_DISPATCH_QUEUED: All work is deferred for a netisr, regardless of
135 * context (may be overriden by protocols).
136 *
137 * NETISR_DISPATCH_HYBRID: If the executing context allows direct dispatch,
138 * and we're running on the CPU the work would be performed on, then direct
139 * dispatch it if it wouldn't violate ordering constraints on the workstream.
140 *
141 * NETISR_DISPATCH_DIRECT: If the executing context allows direct dispatch,
142 * always direct dispatch.  (The default.)
143 *
144 * Notice that changing the global policy could lead to short periods of
145 * misordered processing, but this is considered acceptable as compared to
146 * the complexity of enforcing ordering during policy changes.  Protocols can
147 * override the global policy (when they're not doing that, they select
148 * NETISR_DISPATCH_DEFAULT).
149 */
150#define NETISR_DISPATCH_POLICY_DEFAULT  NETISR_DISPATCH_DIRECT
151#define NETISR_DISPATCH_POLICY_MAXSTR   20 /* Used for temporary buffers. */
152static u_int    netisr_dispatch_policy = NETISR_DISPATCH_POLICY_DEFAULT;
153static int      sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS);
154SYSCTL_PROC(_net_isr, OID_AUTO, dispatch, CTLTYPE_STRING | CTLFLAG_RW |
155    CTLFLAG_TUN, 0, 0, sysctl_netisr_dispatch_policy, "A",
156    "netisr dispatch policy");
157
158/*
159 * These sysctls were used in previous versions to control and export
160 * dispatch policy state.  Now, we provide read-only export via them so that
161 * older netstat binaries work.  At some point they can be garbage collected.
162 */
163static int      netisr_direct_force;
164SYSCTL_INT(_net_isr, OID_AUTO, direct_force, CTLFLAG_RD,
165    &netisr_direct_force, 0, "compat: force direct dispatch");
166
167static int      netisr_direct;
168SYSCTL_INT(_net_isr, OID_AUTO, direct, CTLFLAG_RD, &netisr_direct, 0,
169    "compat: enable direct dispatch");
170
171/*
172 * Allow the administrator to limit the number of threads (CPUs) to use for
173 * netisr.  We don't check netisr_maxthreads before creating the thread for
174 * CPU 0, so in practice we ignore values <= 1.  This must be set at boot.
175 * We will create at most one thread per CPU.
176 */
177static int      netisr_maxthreads = -1;         /* Max number of threads. */
178TUNABLE_INT("net.isr.maxthreads", &netisr_maxthreads);
179SYSCTL_INT(_net_isr, OID_AUTO, maxthreads, CTLFLAG_RDTUN,
180    &netisr_maxthreads, 0,
181    "Use at most this many CPUs for netisr processing");
182
183static int      netisr_bindthreads = 0;         /* Bind threads to CPUs. */
184TUNABLE_INT("net.isr.bindthreads", &netisr_bindthreads);
185SYSCTL_INT(_net_isr, OID_AUTO, bindthreads, CTLFLAG_RDTUN,
186    &netisr_bindthreads, 0, "Bind netisr threads to CPUs.");
187
188/*
189 * Limit per-workstream mbuf queue limits s to at most net.isr.maxqlimit,
190 * both for initial configuration and later modification using
191 * netisr_setqlimit().
192 */
193#define NETISR_DEFAULT_MAXQLIMIT        10240
194static u_int    netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT;
195TUNABLE_INT("net.isr.maxqlimit", &netisr_maxqlimit);
196SYSCTL_UINT(_net_isr, OID_AUTO, maxqlimit, CTLFLAG_RDTUN,
197    &netisr_maxqlimit, 0,
198    "Maximum netisr per-protocol, per-CPU queue depth.");
199
200/*
201 * The default per-workstream mbuf queue limit for protocols that don't
202 * initialize the nh_qlimit field of their struct netisr_handler.  If this is
203 * set above netisr_maxqlimit, we truncate it to the maximum during boot.
204 */
205#define NETISR_DEFAULT_DEFAULTQLIMIT    256
206static u_int    netisr_defaultqlimit = NETISR_DEFAULT_DEFAULTQLIMIT;
207TUNABLE_INT("net.isr.defaultqlimit", &netisr_defaultqlimit);
208SYSCTL_UINT(_net_isr, OID_AUTO, defaultqlimit, CTLFLAG_RDTUN,
209    &netisr_defaultqlimit, 0,
210    "Default netisr per-protocol, per-CPU queue limit if not set by protocol");
211
212/*
213 * Store and export the compile-time constant NETISR_MAXPROT limit on the
214 * number of protocols that can register with netisr at a time.  This is
215 * required for crashdump analysis, as it sizes netisr_proto[].
216 */
217static u_int    netisr_maxprot = NETISR_MAXPROT;
218SYSCTL_UINT(_net_isr, OID_AUTO, maxprot, CTLFLAG_RD,
219    &netisr_maxprot, 0,
220    "Compile-time limit on the number of protocols supported by netisr.");
221
222/*
223 * The netisr_proto array describes all registered protocols, indexed by
224 * protocol number.  See netisr_internal.h for more details.
225 */
226static struct netisr_proto      netisr_proto[NETISR_MAXPROT];
227
228#ifndef __rtems__
229/*
230 * Per-CPU workstream data.  See netisr_internal.h for more details.
231 */
232DPCPU_DEFINE(struct netisr_workstream, nws);
233
234/*
235 * Map contiguous values between 0 and nws_count into CPU IDs appropriate for
236 * accessing workstreams.  This allows constructions of the form
237 * DPCPU_ID_GET(nws_array[arbitraryvalue % nws_count], nws).
238 */
239static u_int                             nws_array[MAXCPU];
240
241/*
242 * Number of registered workstreams.  Will be at most the number of running
243 * CPUs once fully started.
244 */
245static u_int                             nws_count;
246SYSCTL_UINT(_net_isr, OID_AUTO, numthreads, CTLFLAG_RD,
247    &nws_count, 0, "Number of extant netisr threads.");
248#else /* __rtems__ */
249static struct netisr_workstream rtems_bsd_nws;
250#endif /* __rtems__ */
251
252/*
253 * Synchronization for each workstream: a mutex protects all mutable fields
254 * in each stream, including per-protocol state (mbuf queues).  The SWI is
255 * woken up if asynchronous dispatch is required.
256 */
257#define NWS_LOCK(s)             mtx_lock(&(s)->nws_mtx)
258#define NWS_LOCK_ASSERT(s)      mtx_assert(&(s)->nws_mtx, MA_OWNED)
259#define NWS_UNLOCK(s)           mtx_unlock(&(s)->nws_mtx)
260#define NWS_SIGNAL(s)           swi_sched((s)->nws_swi_cookie, 0)
261
262#ifndef __rtems__
263/*
264 * Utility routines for protocols that implement their own mapping of flows
265 * to CPUs.
266 */
267u_int
268netisr_get_cpucount(void)
269{
270
271        return (nws_count);
272}
273
274u_int
275netisr_get_cpuid(u_int cpunumber)
276{
277
278        KASSERT(cpunumber < nws_count, ("%s: %u > %u", __func__, cpunumber,
279            nws_count));
280
281        return (nws_array[cpunumber]);
282}
283
284/*
285 * The default implementation of flow -> CPU ID mapping.
286 *
287 * Non-static so that protocols can use it to map their own work to specific
288 * CPUs in a manner consistent to netisr for affinity purposes.
289 */
290u_int
291netisr_default_flow2cpu(u_int flowid)
292{
293
294        return (nws_array[flowid % nws_count]);
295}
296#endif  /* __rtems__ */
297
298/*
299 * Dispatch tunable and sysctl configuration.
300 */
301struct netisr_dispatch_table_entry {
302        u_int            ndte_policy;
303        const char      *ndte_policy_str;
304};
305static const struct netisr_dispatch_table_entry netisr_dispatch_table[] = {
306        { NETISR_DISPATCH_DEFAULT, "default" },
307        { NETISR_DISPATCH_DEFERRED, "deferred" },
308        { NETISR_DISPATCH_HYBRID, "hybrid" },
309        { NETISR_DISPATCH_DIRECT, "direct" },
310};
311static const u_int netisr_dispatch_table_len =
312    (sizeof(netisr_dispatch_table) / sizeof(netisr_dispatch_table[0]));
313
314static void
315netisr_dispatch_policy_to_str(u_int dispatch_policy, char *buffer,
316    u_int buflen)
317{
318        const struct netisr_dispatch_table_entry *ndtep;
319        const char *str;
320        u_int i;
321
322        str = "unknown";
323        for (i = 0; i < netisr_dispatch_table_len; i++) {
324                ndtep = &netisr_dispatch_table[i];
325                if (ndtep->ndte_policy == dispatch_policy) {
326                        str = ndtep->ndte_policy_str;
327                        break;
328                }
329        }
330        snprintf(buffer, buflen, "%s", str);
331}
332
333static int
334netisr_dispatch_policy_from_str(const char *str, u_int *dispatch_policyp)
335{
336        const struct netisr_dispatch_table_entry *ndtep;
337        u_int i;
338
339        for (i = 0; i < netisr_dispatch_table_len; i++) {
340                ndtep = &netisr_dispatch_table[i];
341                if (strcmp(ndtep->ndte_policy_str, str) == 0) {
342                        *dispatch_policyp = ndtep->ndte_policy;
343                        return (0);
344                }
345        }
346        return (EINVAL);
347}
348
349static void
350netisr_dispatch_policy_compat(void)
351{
352
353        switch (netisr_dispatch_policy) {
354        case NETISR_DISPATCH_DEFERRED:
355                netisr_direct_force = 0;
356                netisr_direct = 0;
357                break;
358
359        case NETISR_DISPATCH_HYBRID:
360                netisr_direct_force = 0;
361                netisr_direct = 1;
362                break;
363
364        case NETISR_DISPATCH_DIRECT:
365                netisr_direct_force = 1;
366                netisr_direct = 1;
367                break;
368
369        default:
370                panic("%s: unknown policy %u", __func__,
371                    netisr_dispatch_policy);
372        }
373}
374
375static int
376sysctl_netisr_dispatch_policy(SYSCTL_HANDLER_ARGS)
377{
378        char tmp[NETISR_DISPATCH_POLICY_MAXSTR];
379        u_int dispatch_policy;
380        int error;
381
382        netisr_dispatch_policy_to_str(netisr_dispatch_policy, tmp,
383            sizeof(tmp));
384        error = sysctl_handle_string(oidp, tmp, sizeof(tmp), req);
385        if (error == 0 && req->newptr != NULL) {
386                error = netisr_dispatch_policy_from_str(tmp,
387                    &dispatch_policy);
388                if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT)
389                        error = EINVAL;
390                if (error == 0) {
391                        netisr_dispatch_policy = dispatch_policy;
392                        netisr_dispatch_policy_compat();
393                }
394        }
395        return (error);
396}
397
398/*
399 * Register a new netisr handler, which requires initializing per-protocol
400 * fields for each workstream.  All netisr work is briefly suspended while
401 * the protocol is installed.
402 */
403void
404netisr_register(const struct netisr_handler *nhp)
405{
406        struct netisr_work *npwp;
407        const char *name;
408        u_int i, proto;
409
410        proto = nhp->nh_proto;
411        name = nhp->nh_name;
412
413        /*
414         * Test that the requested registration is valid.
415         */
416        KASSERT(nhp->nh_name != NULL,
417            ("%s: nh_name NULL for %u", __func__, proto));
418        KASSERT(nhp->nh_handler != NULL,
419            ("%s: nh_handler NULL for %s", __func__, name));
420        KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE ||
421            nhp->nh_policy == NETISR_POLICY_FLOW ||
422            nhp->nh_policy == NETISR_POLICY_CPU,
423            ("%s: unsupported nh_policy %u for %s", __func__,
424            nhp->nh_policy, name));
425        KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW ||
426            nhp->nh_m2flow == NULL,
427            ("%s: nh_policy != FLOW but m2flow defined for %s", __func__,
428            name));
429        KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpuid == NULL,
430            ("%s: nh_policy != CPU but m2cpuid defined for %s", __func__,
431            name));
432        KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpuid != NULL,
433            ("%s: nh_policy == CPU but m2cpuid not defined for %s", __func__,
434            name));
435        KASSERT(nhp->nh_dispatch == NETISR_DISPATCH_DEFAULT ||
436            nhp->nh_dispatch == NETISR_DISPATCH_DEFERRED ||
437            nhp->nh_dispatch == NETISR_DISPATCH_HYBRID ||
438            nhp->nh_dispatch == NETISR_DISPATCH_DIRECT,
439            ("%s: invalid nh_dispatch (%u)", __func__, nhp->nh_dispatch));
440
441        KASSERT(proto < NETISR_MAXPROT,
442            ("%s(%u, %s): protocol too big", __func__, proto, name));
443
444        /*
445         * Test that no existing registration exists for this protocol.
446         */
447        NETISR_WLOCK();
448        KASSERT(netisr_proto[proto].np_name == NULL,
449            ("%s(%u, %s): name present", __func__, proto, name));
450        KASSERT(netisr_proto[proto].np_handler == NULL,
451            ("%s(%u, %s): handler present", __func__, proto, name));
452
453        netisr_proto[proto].np_name = name;
454        netisr_proto[proto].np_handler = nhp->nh_handler;
455        netisr_proto[proto].np_m2flow = nhp->nh_m2flow;
456        netisr_proto[proto].np_m2cpuid = nhp->nh_m2cpuid;
457        netisr_proto[proto].np_drainedcpu = nhp->nh_drainedcpu;
458        if (nhp->nh_qlimit == 0)
459                netisr_proto[proto].np_qlimit = netisr_defaultqlimit;
460        else if (nhp->nh_qlimit > netisr_maxqlimit) {
461                printf("%s: %s requested queue limit %u capped to "
462                    "net.isr.maxqlimit %u\n", __func__, name, nhp->nh_qlimit,
463                    netisr_maxqlimit);
464                netisr_proto[proto].np_qlimit = netisr_maxqlimit;
465        } else
466                netisr_proto[proto].np_qlimit = nhp->nh_qlimit;
467        netisr_proto[proto].np_policy = nhp->nh_policy;
468        netisr_proto[proto].np_dispatch = nhp->nh_dispatch;
469        CPU_FOREACH(i) {
470#ifndef __rtems__
471                npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto];
472#else /* __rtems__ */
473                npwp = &rtems_bsd_nws.nws_work[proto];
474#endif /* __rtems__ */
475                bzero(npwp, sizeof(*npwp));
476                npwp->nw_qlimit = netisr_proto[proto].np_qlimit;
477        }
478        NETISR_WUNLOCK();
479}
480
481/*
482 * Clear drop counters across all workstreams for a protocol.
483 */
484void
485netisr_clearqdrops(const struct netisr_handler *nhp)
486{
487        struct netisr_work *npwp;
488#ifdef INVARIANTS
489        const char *name;
490#endif
491        u_int i, proto;
492
493        proto = nhp->nh_proto;
494#ifdef INVARIANTS
495        name = nhp->nh_name;
496#endif
497        KASSERT(proto < NETISR_MAXPROT,
498            ("%s(%u): protocol too big for %s", __func__, proto, name));
499
500        NETISR_WLOCK();
501        KASSERT(netisr_proto[proto].np_handler != NULL,
502            ("%s(%u): protocol not registered for %s", __func__, proto,
503            name));
504
505        CPU_FOREACH(i) {
506#ifndef __rtems__
507                npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto];
508#else /* __rtems__ */
509                npwp = &rtems_bsd_nws.nws_work[proto];
510#endif /* __rtems__ */
511                npwp->nw_qdrops = 0;
512        }
513        NETISR_WUNLOCK();
514}
515
516/*
517 * Query current drop counters across all workstreams for a protocol.
518 */
519void
520netisr_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp)
521{
522        struct netisr_work *npwp;
523        struct rm_priotracker tracker;
524#ifdef INVARIANTS
525        const char *name;
526#endif
527        u_int i, proto;
528
529        *qdropp = 0;
530        proto = nhp->nh_proto;
531#ifdef INVARIANTS
532        name = nhp->nh_name;
533#endif
534        KASSERT(proto < NETISR_MAXPROT,
535            ("%s(%u): protocol too big for %s", __func__, proto, name));
536
537        NETISR_RLOCK(&tracker);
538        KASSERT(netisr_proto[proto].np_handler != NULL,
539            ("%s(%u): protocol not registered for %s", __func__, proto,
540            name));
541
542        CPU_FOREACH(i) {
543#ifndef __rtems__
544                npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto];
545#else /* __rtems__ */
546                npwp = &rtems_bsd_nws.nws_work[proto];
547#endif /* __rtems__ */
548                *qdropp += npwp->nw_qdrops;
549        }
550        NETISR_RUNLOCK(&tracker);
551}
552
553/*
554 * Query current per-workstream queue limit for a protocol.
555 */
556void
557netisr_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp)
558{
559        struct rm_priotracker tracker;
560#ifdef INVARIANTS
561        const char *name;
562#endif
563        u_int proto;
564
565        proto = nhp->nh_proto;
566#ifdef INVARIANTS
567        name = nhp->nh_name;
568#endif
569        KASSERT(proto < NETISR_MAXPROT,
570            ("%s(%u): protocol too big for %s", __func__, proto, name));
571
572        NETISR_RLOCK(&tracker);
573        KASSERT(netisr_proto[proto].np_handler != NULL,
574            ("%s(%u): protocol not registered for %s", __func__, proto,
575            name));
576        *qlimitp = netisr_proto[proto].np_qlimit;
577        NETISR_RUNLOCK(&tracker);
578}
579
580/*
581 * Update the queue limit across per-workstream queues for a protocol.  We
582 * simply change the limits, and don't drain overflowed packets as they will
583 * (hopefully) take care of themselves shortly.
584 */
585int
586netisr_setqlimit(const struct netisr_handler *nhp, u_int qlimit)
587{
588        struct netisr_work *npwp;
589#ifdef INVARIANTS
590        const char *name;
591#endif
592        u_int i, proto;
593
594        if (qlimit > netisr_maxqlimit)
595                return (EINVAL);
596
597        proto = nhp->nh_proto;
598#ifdef INVARIANTS
599        name = nhp->nh_name;
600#endif
601        KASSERT(proto < NETISR_MAXPROT,
602            ("%s(%u): protocol too big for %s", __func__, proto, name));
603
604        NETISR_WLOCK();
605        KASSERT(netisr_proto[proto].np_handler != NULL,
606            ("%s(%u): protocol not registered for %s", __func__, proto,
607            name));
608
609        netisr_proto[proto].np_qlimit = qlimit;
610        CPU_FOREACH(i) {
611#ifndef __rtems__
612                npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto];
613#else /* __rtems__ */
614                npwp = &rtems_bsd_nws.nws_work[proto];
615#endif /* __rtems__ */
616                npwp->nw_qlimit = qlimit;
617        }
618        NETISR_WUNLOCK();
619        return (0);
620}
621
622/*
623 * Drain all packets currently held in a particular protocol work queue.
624 */
625static void
626netisr_drain_proto(struct netisr_work *npwp)
627{
628        struct mbuf *m;
629
630        /*
631         * We would assert the lock on the workstream but it's not passed in.
632         */
633        while ((m = npwp->nw_head) != NULL) {
634                npwp->nw_head = m->m_nextpkt;
635                m->m_nextpkt = NULL;
636                if (npwp->nw_head == NULL)
637                        npwp->nw_tail = NULL;
638                npwp->nw_len--;
639                m_freem(m);
640        }
641        KASSERT(npwp->nw_tail == NULL, ("%s: tail", __func__));
642        KASSERT(npwp->nw_len == 0, ("%s: len", __func__));
643}
644
645/*
646 * Remove the registration of a network protocol, which requires clearing
647 * per-protocol fields across all workstreams, including freeing all mbufs in
648 * the queues at time of unregister.  All work in netisr is briefly suspended
649 * while this takes place.
650 */
651void
652netisr_unregister(const struct netisr_handler *nhp)
653{
654        struct netisr_work *npwp;
655#ifdef INVARIANTS
656        const char *name;
657#endif
658        u_int i, proto;
659
660        proto = nhp->nh_proto;
661#ifdef INVARIANTS
662        name = nhp->nh_name;
663#endif
664        KASSERT(proto < NETISR_MAXPROT,
665            ("%s(%u): protocol too big for %s", __func__, proto, name));
666
667        NETISR_WLOCK();
668        KASSERT(netisr_proto[proto].np_handler != NULL,
669            ("%s(%u): protocol not registered for %s", __func__, proto,
670            name));
671
672        netisr_proto[proto].np_name = NULL;
673        netisr_proto[proto].np_handler = NULL;
674        netisr_proto[proto].np_m2flow = NULL;
675        netisr_proto[proto].np_m2cpuid = NULL;
676        netisr_proto[proto].np_qlimit = 0;
677        netisr_proto[proto].np_policy = 0;
678        CPU_FOREACH(i) {
679#ifndef __rtems__
680                npwp = &(DPCPU_ID_PTR(i, nws))->nws_work[proto];
681#else /* __rtems__ */
682                npwp = &rtems_bsd_nws.nws_work[proto];
683#endif /* __rtems__ */
684                netisr_drain_proto(npwp);
685                bzero(npwp, sizeof(*npwp));
686        }
687        NETISR_WUNLOCK();
688}
689
690/*
691 * Compose the global and per-protocol policies on dispatch, and return the
692 * dispatch policy to use.
693 */
694static u_int
695netisr_get_dispatch(struct netisr_proto *npp)
696{
697
698        /*
699         * Protocol-specific configuration overrides the global default.
700         */
701        if (npp->np_dispatch != NETISR_DISPATCH_DEFAULT)
702                return (npp->np_dispatch);
703        return (netisr_dispatch_policy);
704}
705
706/*
707 * Look up the workstream given a packet and source identifier.  Do this by
708 * checking the protocol's policy, and optionally call out to the protocol
709 * for assistance if required.
710 */
711static struct mbuf *
712netisr_select_cpuid(struct netisr_proto *npp, u_int dispatch_policy,
713    uintptr_t source, struct mbuf *m, u_int *cpuidp)
714{
715        struct ifnet *ifp;
716        u_int policy;
717
718        NETISR_LOCK_ASSERT();
719
720#ifndef __rtems__
721        /*
722         * In the event we have only one worker, shortcut and deliver to it
723         * without further ado.
724         */
725        if (nws_count == 1) {
726                *cpuidp = nws_array[0];
727                return (m);
728        }
729
730        /*
731         * What happens next depends on the policy selected by the protocol.
732         * If we want to support per-interface policies, we should do that
733         * here first.
734         */
735        policy = npp->np_policy;
736        if (policy == NETISR_POLICY_CPU) {
737                m = npp->np_m2cpuid(m, source, cpuidp);
738                if (m == NULL)
739                        return (NULL);
740
741                /*
742                 * It's possible for a protocol not to have a good idea about
743                 * where to process a packet, in which case we fall back on
744                 * the netisr code to decide.  In the hybrid case, return the
745                 * current CPU ID, which will force an immediate direct
746                 * dispatch.  In the queued case, fall back on the SOURCE
747                 * policy.
748                 */
749                if (*cpuidp != NETISR_CPUID_NONE)
750                        return (m);
751                if (dispatch_policy == NETISR_DISPATCH_HYBRID) {
752                        *cpuidp = curcpu;
753                        return (m);
754                }
755                policy = NETISR_POLICY_SOURCE;
756        }
757
758        if (policy == NETISR_POLICY_FLOW) {
759                if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) {
760                        m = npp->np_m2flow(m, source);
761                        if (m == NULL)
762                                return (NULL);
763                }
764                if (m->m_flags & M_FLOWID) {
765                        *cpuidp =
766                            netisr_default_flow2cpu(m->m_pkthdr.flowid);
767                        return (m);
768                }
769                policy = NETISR_POLICY_SOURCE;
770        }
771
772        KASSERT(policy == NETISR_POLICY_SOURCE,
773            ("%s: invalid policy %u for %s", __func__, npp->np_policy,
774            npp->np_name));
775
776        ifp = m->m_pkthdr.rcvif;
777        if (ifp != NULL)
778                *cpuidp = nws_array[(ifp->if_index + source) % nws_count];
779        else
780                *cpuidp = nws_array[source % nws_count];
781#else /* __rtems__ */
782        *cpuidp = 0;
783#endif /* __rtems__ */
784        return (m);
785}
786
787/*
788 * Process packets associated with a workstream and protocol.  For reasons of
789 * fairness, we process up to one complete netisr queue at a time, moving the
790 * queue to a stack-local queue for processing, but do not loop refreshing
791 * from the global queue.  The caller is responsible for deciding whether to
792 * loop, and for setting the NWS_RUNNING flag.  The passed workstream will be
793 * locked on entry and relocked before return, but will be released while
794 * processing.  The number of packets processed is returned.
795 */
796static u_int
797netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto)
798{
799        struct netisr_work local_npw, *npwp;
800        u_int handled;
801        struct mbuf *m;
802
803        NETISR_LOCK_ASSERT();
804        NWS_LOCK_ASSERT(nwsp);
805
806        KASSERT(nwsp->nws_flags & NWS_RUNNING,
807            ("%s(%u): not running", __func__, proto));
808        KASSERT(proto >= 0 && proto < NETISR_MAXPROT,
809            ("%s(%u): invalid proto\n", __func__, proto));
810
811        npwp = &nwsp->nws_work[proto];
812        if (npwp->nw_len == 0)
813                return (0);
814
815        /*
816         * Move the global work queue to a thread-local work queue.
817         *
818         * Notice that this means the effective maximum length of the queue
819         * is actually twice that of the maximum queue length specified in
820         * the protocol registration call.
821         */
822        handled = npwp->nw_len;
823        local_npw = *npwp;
824        npwp->nw_head = NULL;
825        npwp->nw_tail = NULL;
826        npwp->nw_len = 0;
827        nwsp->nws_pendingbits &= ~(1 << proto);
828        NWS_UNLOCK(nwsp);
829        while ((m = local_npw.nw_head) != NULL) {
830                local_npw.nw_head = m->m_nextpkt;
831                m->m_nextpkt = NULL;
832                if (local_npw.nw_head == NULL)
833                        local_npw.nw_tail = NULL;
834                local_npw.nw_len--;
835                VNET_ASSERT(m->m_pkthdr.rcvif != NULL,
836                    ("%s:%d rcvif == NULL: m=%p", __func__, __LINE__, m));
837                CURVNET_SET(m->m_pkthdr.rcvif->if_vnet);
838                netisr_proto[proto].np_handler(m);
839                CURVNET_RESTORE();
840        }
841        KASSERT(local_npw.nw_len == 0,
842            ("%s(%u): len %u", __func__, proto, local_npw.nw_len));
843        if (netisr_proto[proto].np_drainedcpu)
844                netisr_proto[proto].np_drainedcpu(nwsp->nws_cpu);
845        NWS_LOCK(nwsp);
846        npwp->nw_handled += handled;
847        return (handled);
848}
849
850/*
851 * SWI handler for netisr -- processes packets in a set of workstreams that
852 * it owns, woken up by calls to NWS_SIGNAL().  If this workstream is already
853 * being direct dispatched, go back to sleep and wait for the dispatching
854 * thread to wake us up again.
855 */
856static void
857swi_net(void *arg)
858{
859#ifdef NETISR_LOCKING
860        struct rm_priotracker tracker;
861#endif
862        struct netisr_workstream *nwsp;
863        u_int bits, prot;
864
865        nwsp = arg;
866
867#ifdef DEVICE_POLLING
868        KASSERT(nws_count == 1,
869            ("%s: device_polling but nws_count != 1", __func__));
870        netisr_poll();
871#endif
872#ifdef NETISR_LOCKING
873        NETISR_RLOCK(&tracker);
874#endif
875        NWS_LOCK(nwsp);
876        KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running"));
877        if (nwsp->nws_flags & NWS_DISPATCHING)
878                goto out;
879        nwsp->nws_flags |= NWS_RUNNING;
880        nwsp->nws_flags &= ~NWS_SCHEDULED;
881        while ((bits = nwsp->nws_pendingbits) != 0) {
882                while ((prot = ffs(bits)) != 0) {
883                        prot--;
884                        bits &= ~(1 << prot);
885                        (void)netisr_process_workstream_proto(nwsp, prot);
886                }
887        }
888        nwsp->nws_flags &= ~NWS_RUNNING;
889out:
890        NWS_UNLOCK(nwsp);
891#ifdef NETISR_LOCKING
892        NETISR_RUNLOCK(&tracker);
893#endif
894#ifdef DEVICE_POLLING
895        netisr_pollmore();
896#endif
897}
898
899static int
900netisr_queue_workstream(struct netisr_workstream *nwsp, u_int proto,
901    struct netisr_work *npwp, struct mbuf *m, int *dosignalp)
902{
903
904        NWS_LOCK_ASSERT(nwsp);
905
906        *dosignalp = 0;
907        if (npwp->nw_len < npwp->nw_qlimit) {
908                m->m_nextpkt = NULL;
909                if (npwp->nw_head == NULL) {
910                        npwp->nw_head = m;
911                        npwp->nw_tail = m;
912                } else {
913                        npwp->nw_tail->m_nextpkt = m;
914                        npwp->nw_tail = m;
915                }
916                npwp->nw_len++;
917                if (npwp->nw_len > npwp->nw_watermark)
918                        npwp->nw_watermark = npwp->nw_len;
919
920                /*
921                 * We must set the bit regardless of NWS_RUNNING, so that
922                 * swi_net() keeps calling netisr_process_workstream_proto().
923                 */
924                nwsp->nws_pendingbits |= (1 << proto);
925                if (!(nwsp->nws_flags & 
926                    (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED))) {
927                        nwsp->nws_flags |= NWS_SCHEDULED;
928                        *dosignalp = 1; /* Defer until unlocked. */
929                }
930                npwp->nw_queued++;
931                return (0);
932        } else {
933                m_freem(m);
934                npwp->nw_qdrops++;
935                return (ENOBUFS);
936        }
937}
938
939static int
940netisr_queue_internal(u_int proto, struct mbuf *m, u_int cpuid)
941{
942        struct netisr_workstream *nwsp;
943        struct netisr_work *npwp;
944        int dosignal, error;
945
946#ifdef NETISR_LOCKING
947        NETISR_LOCK_ASSERT();
948#endif
949        KASSERT(cpuid <= mp_maxid, ("%s: cpuid too big (%u, %u)", __func__,
950            cpuid, mp_maxid));
951        KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid));
952
953        dosignal = 0;
954        error = 0;
955#ifndef __rtems__
956        nwsp = DPCPU_ID_PTR(cpuid, nws);
957#else /* __rtems__ */
958        nwsp = &rtems_bsd_nws;
959#endif /* __rtems__ */
960        npwp = &nwsp->nws_work[proto];
961        NWS_LOCK(nwsp);
962        error = netisr_queue_workstream(nwsp, proto, npwp, m, &dosignal);
963        NWS_UNLOCK(nwsp);
964        if (dosignal)
965                NWS_SIGNAL(nwsp);
966        return (error);
967}
968
969int
970netisr_queue_src(u_int proto, uintptr_t source, struct mbuf *m)
971{
972#ifdef NETISR_LOCKING
973        struct rm_priotracker tracker;
974#endif
975        u_int cpuid;
976        int error;
977
978        KASSERT(proto < NETISR_MAXPROT,
979            ("%s: invalid proto %u", __func__, proto));
980
981#ifdef NETISR_LOCKING
982        NETISR_RLOCK(&tracker);
983#endif
984        KASSERT(netisr_proto[proto].np_handler != NULL,
985            ("%s: invalid proto %u", __func__, proto));
986
987        m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_DEFERRED,
988            source, m, &cpuid);
989        if (m != NULL) {
990                KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__,
991                    cpuid));
992                error = netisr_queue_internal(proto, m, cpuid);
993        } else
994                error = ENOBUFS;
995#ifdef NETISR_LOCKING
996        NETISR_RUNLOCK(&tracker);
997#endif
998        return (error);
999}
1000
1001int
1002netisr_queue(u_int proto, struct mbuf *m)
1003{
1004
1005        return (netisr_queue_src(proto, 0, m));
1006}
1007
1008/*
1009 * Dispatch a packet for netisr processing; direct dispatch is permitted by
1010 * calling context.
1011 */
1012int
1013netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m)
1014{
1015#ifdef NETISR_LOCKING
1016        struct rm_priotracker tracker;
1017#endif
1018        struct netisr_workstream *nwsp;
1019        struct netisr_proto *npp;
1020        struct netisr_work *npwp;
1021        int dosignal, error;
1022        u_int cpuid, dispatch_policy;
1023
1024        KASSERT(proto < NETISR_MAXPROT,
1025            ("%s: invalid proto %u", __func__, proto));
1026#ifdef NETISR_LOCKING
1027        NETISR_RLOCK(&tracker);
1028#endif
1029        npp = &netisr_proto[proto];
1030        KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__,
1031            proto));
1032
1033        dispatch_policy = netisr_get_dispatch(npp);
1034        if (dispatch_policy == NETISR_DISPATCH_DEFERRED)
1035                return (netisr_queue_src(proto, source, m));
1036
1037        /*
1038         * If direct dispatch is forced, then unconditionally dispatch
1039         * without a formal CPU selection.  Borrow the current CPU's stats,
1040         * even if there's no worker on it.  In this case we don't update
1041         * nws_flags because all netisr processing will be source ordered due
1042         * to always being forced to directly dispatch.
1043         */
1044        if (dispatch_policy == NETISR_DISPATCH_DIRECT) {
1045#ifndef __rtems__
1046                nwsp = DPCPU_PTR(nws);
1047#else /* __rtems__ */
1048                nwsp = &rtems_bsd_nws;
1049#endif /* __rtems__ */
1050                npwp = &nwsp->nws_work[proto];
1051                npwp->nw_dispatched++;
1052                npwp->nw_handled++;
1053                netisr_proto[proto].np_handler(m);
1054                error = 0;
1055                goto out_unlock;
1056        }
1057
1058        KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID,
1059            ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy));
1060
1061        /*
1062         * Otherwise, we execute in a hybrid mode where we will try to direct
1063         * dispatch if we're on the right CPU and the netisr worker isn't
1064         * already running.
1065         */
1066#ifndef __rtems__
1067        sched_pin();
1068#endif /* __rtems__ */
1069        m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_HYBRID,
1070            source, m, &cpuid);
1071        if (m == NULL) {
1072                error = ENOBUFS;
1073                goto out_unpin;
1074        }
1075        KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid));
1076        if (cpuid != curcpu)
1077                goto queue_fallback;
1078#ifndef __rtems__
1079        nwsp = DPCPU_PTR(nws);
1080#else /* __rtems__ */
1081        nwsp = &rtems_bsd_nws;
1082#endif /* __rtems__ */
1083        npwp = &nwsp->nws_work[proto];
1084
1085        /*-
1086         * We are willing to direct dispatch only if three conditions hold:
1087         *
1088         * (1) The netisr worker isn't already running,
1089         * (2) Another thread isn't already directly dispatching, and
1090         * (3) The netisr hasn't already been woken up.
1091         */
1092        NWS_LOCK(nwsp);
1093        if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) {
1094                error = netisr_queue_workstream(nwsp, proto, npwp, m,
1095                    &dosignal);
1096                NWS_UNLOCK(nwsp);
1097                if (dosignal)
1098                        NWS_SIGNAL(nwsp);
1099                goto out_unpin;
1100        }
1101
1102        /*
1103         * The current thread is now effectively the netisr worker, so set
1104         * the dispatching flag to prevent concurrent processing of the
1105         * stream from another thread (even the netisr worker), which could
1106         * otherwise lead to effective misordering of the stream.
1107         */
1108        nwsp->nws_flags |= NWS_DISPATCHING;
1109        NWS_UNLOCK(nwsp);
1110        netisr_proto[proto].np_handler(m);
1111        NWS_LOCK(nwsp);
1112        nwsp->nws_flags &= ~NWS_DISPATCHING;
1113        npwp->nw_handled++;
1114        npwp->nw_hybrid_dispatched++;
1115
1116        /*
1117         * If other work was enqueued by another thread while we were direct
1118         * dispatching, we need to signal the netisr worker to do that work.
1119         * In the future, we might want to do some of that work in the
1120         * current thread, rather than trigger further context switches.  If
1121         * so, we'll want to establish a reasonable bound on the work done in
1122         * the "borrowed" context.
1123         */
1124        if (nwsp->nws_pendingbits != 0) {
1125                nwsp->nws_flags |= NWS_SCHEDULED;
1126                dosignal = 1;
1127        } else
1128                dosignal = 0;
1129        NWS_UNLOCK(nwsp);
1130        if (dosignal)
1131                NWS_SIGNAL(nwsp);
1132        error = 0;
1133        goto out_unpin;
1134
1135queue_fallback:
1136        error = netisr_queue_internal(proto, m, cpuid);
1137out_unpin:
1138#ifndef __rtems__
1139        sched_unpin();
1140#endif  /* __rtems__ */
1141out_unlock:
1142#ifdef NETISR_LOCKING
1143        NETISR_RUNLOCK(&tracker);
1144#endif
1145        return (error);
1146}
1147
1148int
1149netisr_dispatch(u_int proto, struct mbuf *m)
1150{
1151
1152        return (netisr_dispatch_src(proto, 0, m));
1153}
1154
1155#ifdef DEVICE_POLLING
1156/*
1157 * Kernel polling borrows a netisr thread to run interface polling in; this
1158 * function allows kernel polling to request that the netisr thread be
1159 * scheduled even if no packets are pending for protocols.
1160 */
1161void
1162netisr_sched_poll(void)
1163{
1164        struct netisr_workstream *nwsp;
1165
1166        nwsp = DPCPU_ID_PTR(nws_array[0], nws);
1167        NWS_SIGNAL(nwsp);
1168}
1169#endif
1170
1171static void
1172netisr_start_swi(u_int cpuid, struct pcpu *pc)
1173{
1174        char swiname[12];
1175        struct netisr_workstream *nwsp;
1176        int error;
1177
1178        KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid));
1179
1180#ifndef __rtems__
1181        nwsp = DPCPU_ID_PTR(cpuid, nws);
1182#else /* __rtems__ */
1183        nwsp = &rtems_bsd_nws;
1184#endif /* __rtems__ */
1185        mtx_init(&nwsp->nws_mtx, "netisr_mtx", NULL, MTX_DEF);
1186        nwsp->nws_cpu = cpuid;
1187        snprintf(swiname, sizeof(swiname), "netisr %u", cpuid);
1188        error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp,
1189            SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie);
1190        if (error)
1191                panic("%s: swi_add %d", __func__, error);
1192#ifndef __rtems__
1193        pc->pc_netisr = nwsp->nws_intr_event;
1194        if (netisr_bindthreads) {
1195                error = intr_event_bind(nwsp->nws_intr_event, cpuid);
1196                if (error != 0)
1197                        printf("%s: cpu %u: intr_event_bind: %d", __func__,
1198                            cpuid, error);
1199        }
1200        NETISR_WLOCK();
1201        nws_array[nws_count] = nwsp->nws_cpu;
1202        nws_count++;
1203        NETISR_WUNLOCK();
1204#endif /* __rtems__ */
1205}
1206
1207/*
1208 * Initialize the netisr subsystem.  We rely on BSS and static initialization
1209 * of most fields in global data structures.
1210 *
1211 * Start a worker thread for the boot CPU so that we can support network
1212 * traffic immediately in case the network stack is used before additional
1213 * CPUs are started (for example, diskless boot).
1214 */
1215static void
1216netisr_init(void *arg)
1217{
1218        char tmp[NETISR_DISPATCH_POLICY_MAXSTR];
1219        u_int dispatch_policy;
1220        int error;
1221
1222        KASSERT(curcpu == 0, ("%s: not on CPU 0", __func__));
1223
1224        NETISR_LOCK_INIT();
1225        if (netisr_maxthreads < 1)
1226                netisr_maxthreads = 1;
1227        if (netisr_maxthreads > mp_ncpus) {
1228                printf("netisr_init: forcing maxthreads from %d to %d\n",
1229                    netisr_maxthreads, mp_ncpus);
1230                netisr_maxthreads = mp_ncpus;
1231        }
1232        if (netisr_defaultqlimit > netisr_maxqlimit) {
1233                printf("netisr_init: forcing defaultqlimit from %d to %d\n",
1234                    netisr_defaultqlimit, netisr_maxqlimit);
1235                netisr_defaultqlimit = netisr_maxqlimit;
1236        }
1237#ifdef DEVICE_POLLING
1238        /*
1239         * The device polling code is not yet aware of how to deal with
1240         * multiple netisr threads, so for the time being compiling in device
1241         * polling disables parallel netisr workers.
1242         */
1243        if (netisr_maxthreads != 1 || netisr_bindthreads != 0) {
1244                printf("netisr_init: forcing maxthreads to 1 and "
1245                    "bindthreads to 0 for device polling\n");
1246                netisr_maxthreads = 1;
1247                netisr_bindthreads = 0;
1248        }
1249#endif
1250
1251#ifndef __rtems__
1252        if (TUNABLE_STR_FETCH("net.isr.dispatch", tmp, sizeof(tmp))) {
1253                error = netisr_dispatch_policy_from_str(tmp,
1254                    &dispatch_policy);
1255                if (error == 0 && dispatch_policy == NETISR_DISPATCH_DEFAULT)
1256                        error = EINVAL;
1257                if (error == 0) {
1258                        netisr_dispatch_policy = dispatch_policy;
1259                        netisr_dispatch_policy_compat();
1260                } else
1261                        printf(
1262                            "%s: invalid dispatch policy %s, using default\n",
1263                            __func__, tmp);
1264        }
1265#endif /* __rtems__ */
1266
1267#ifndef __rtems__
1268        netisr_start_swi(curcpu, pcpu_find(curcpu));
1269#else /* __rtems__ */
1270        netisr_start_swi(0, NULL);
1271#endif /* __rtems__ */
1272}
1273SYSINIT(netisr_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr_init, NULL);
1274
1275#ifndef __rtems__
1276/*
1277 * Start worker threads for additional CPUs.  No attempt to gracefully handle
1278 * work reassignment, we don't yet support dynamic reconfiguration.
1279 */
1280static void
1281netisr_start(void *arg)
1282{
1283        struct pcpu *pc;
1284
1285        STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
1286                if (nws_count >= netisr_maxthreads)
1287                        break;
1288                /* XXXRW: Is skipping absent CPUs still required here? */
1289                if (CPU_ABSENT(pc->pc_cpuid))
1290                        continue;
1291                /* Worker will already be present for boot CPU. */
1292                if (pc->pc_netisr != NULL)
1293                        continue;
1294                netisr_start_swi(pc->pc_cpuid, pc);
1295        }
1296}
1297SYSINIT(netisr_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr_start, NULL);
1298#endif /* __rtems__ */
1299
1300/*
1301 * Sysctl monitoring for netisr: query a list of registered protocols.
1302 */
1303static int
1304sysctl_netisr_proto(SYSCTL_HANDLER_ARGS)
1305{
1306        struct rm_priotracker tracker;
1307        struct sysctl_netisr_proto *snpp, *snp_array;
1308        struct netisr_proto *npp;
1309        u_int counter, proto;
1310        int error;
1311
1312        if (req->newptr != NULL)
1313                return (EINVAL);
1314        snp_array = malloc(sizeof(*snp_array) * NETISR_MAXPROT, M_TEMP,
1315            M_ZERO | M_WAITOK);
1316        counter = 0;
1317        NETISR_RLOCK(&tracker);
1318        for (proto = 0; proto < NETISR_MAXPROT; proto++) {
1319                npp = &netisr_proto[proto];
1320                if (npp->np_name == NULL)
1321                        continue;
1322                snpp = &snp_array[counter];
1323                snpp->snp_version = sizeof(*snpp);
1324                strlcpy(snpp->snp_name, npp->np_name, NETISR_NAMEMAXLEN);
1325                snpp->snp_proto = proto;
1326                snpp->snp_qlimit = npp->np_qlimit;
1327                snpp->snp_policy = npp->np_policy;
1328                snpp->snp_dispatch = npp->np_dispatch;
1329                if (npp->np_m2flow != NULL)
1330                        snpp->snp_flags |= NETISR_SNP_FLAGS_M2FLOW;
1331                if (npp->np_m2cpuid != NULL)
1332                        snpp->snp_flags |= NETISR_SNP_FLAGS_M2CPUID;
1333                if (npp->np_drainedcpu != NULL)
1334                        snpp->snp_flags |= NETISR_SNP_FLAGS_DRAINEDCPU;
1335                counter++;
1336        }
1337        NETISR_RUNLOCK(&tracker);
1338        KASSERT(counter <= NETISR_MAXPROT,
1339            ("sysctl_netisr_proto: counter too big (%d)", counter));
1340        error = SYSCTL_OUT(req, snp_array, sizeof(*snp_array) * counter);
1341        free(snp_array, M_TEMP);
1342        return (error);
1343}
1344
1345SYSCTL_PROC(_net_isr, OID_AUTO, proto,
1346    CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_proto,
1347    "S,sysctl_netisr_proto",
1348    "Return list of protocols registered with netisr");
1349
1350/*
1351 * Sysctl monitoring for netisr: query a list of workstreams.
1352 */
1353static int
1354sysctl_netisr_workstream(SYSCTL_HANDLER_ARGS)
1355{
1356        struct rm_priotracker tracker;
1357        struct sysctl_netisr_workstream *snwsp, *snws_array;
1358        struct netisr_workstream *nwsp;
1359        u_int counter, cpuid;
1360        int error;
1361
1362        if (req->newptr != NULL)
1363                return (EINVAL);
1364        snws_array = malloc(sizeof(*snws_array) * MAXCPU, M_TEMP,
1365            M_ZERO | M_WAITOK);
1366        counter = 0;
1367        NETISR_RLOCK(&tracker);
1368        CPU_FOREACH(cpuid) {
1369                nwsp = DPCPU_ID_PTR(cpuid, nws);
1370                if (nwsp->nws_intr_event == NULL)
1371                        continue;
1372                NWS_LOCK(nwsp);
1373                snwsp = &snws_array[counter];
1374                snwsp->snws_version = sizeof(*snwsp);
1375
1376                /*
1377                 * For now, we equate workstream IDs and CPU IDs in the
1378                 * kernel, but expose them independently to userspace in case
1379                 * that assumption changes in the future.
1380                 */
1381                snwsp->snws_wsid = cpuid;
1382                snwsp->snws_cpu = cpuid;
1383                if (nwsp->nws_intr_event != NULL)
1384                        snwsp->snws_flags |= NETISR_SNWS_FLAGS_INTR;
1385                NWS_UNLOCK(nwsp);
1386                counter++;
1387        }
1388        NETISR_RUNLOCK(&tracker);
1389        KASSERT(counter <= MAXCPU,
1390            ("sysctl_netisr_workstream: counter too big (%d)", counter));
1391        error = SYSCTL_OUT(req, snws_array, sizeof(*snws_array) * counter);
1392        free(snws_array, M_TEMP);
1393        return (error);
1394}
1395
1396SYSCTL_PROC(_net_isr, OID_AUTO, workstream,
1397    CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_workstream,
1398    "S,sysctl_netisr_workstream",
1399    "Return list of workstreams implemented by netisr");
1400
1401/*
1402 * Sysctl monitoring for netisr: query per-protocol data across all
1403 * workstreams.
1404 */
1405static int
1406sysctl_netisr_work(SYSCTL_HANDLER_ARGS)
1407{
1408        struct rm_priotracker tracker;
1409        struct sysctl_netisr_work *snwp, *snw_array;
1410        struct netisr_workstream *nwsp;
1411        struct netisr_proto *npp;
1412        struct netisr_work *nwp;
1413        u_int counter, cpuid, proto;
1414        int error;
1415
1416        if (req->newptr != NULL)
1417                return (EINVAL);
1418        snw_array = malloc(sizeof(*snw_array) * MAXCPU * NETISR_MAXPROT,
1419            M_TEMP, M_ZERO | M_WAITOK);
1420        counter = 0;
1421        NETISR_RLOCK(&tracker);
1422        CPU_FOREACH(cpuid) {
1423                nwsp = DPCPU_ID_PTR(cpuid, nws);
1424                if (nwsp->nws_intr_event == NULL)
1425                        continue;
1426                NWS_LOCK(nwsp);
1427                for (proto = 0; proto < NETISR_MAXPROT; proto++) {
1428                        npp = &netisr_proto[proto];
1429                        if (npp->np_name == NULL)
1430                                continue;
1431                        nwp = &nwsp->nws_work[proto];
1432                        snwp = &snw_array[counter];
1433                        snwp->snw_version = sizeof(*snwp);
1434                        snwp->snw_wsid = cpuid;         /* See comment above. */
1435                        snwp->snw_proto = proto;
1436                        snwp->snw_len = nwp->nw_len;
1437                        snwp->snw_watermark = nwp->nw_watermark;
1438                        snwp->snw_dispatched = nwp->nw_dispatched;
1439                        snwp->snw_hybrid_dispatched =
1440                            nwp->nw_hybrid_dispatched;
1441                        snwp->snw_qdrops = nwp->nw_qdrops;
1442                        snwp->snw_queued = nwp->nw_queued;
1443                        snwp->snw_handled = nwp->nw_handled;
1444                        counter++;
1445                }
1446                NWS_UNLOCK(nwsp);
1447        }
1448        KASSERT(counter <= MAXCPU * NETISR_MAXPROT,
1449            ("sysctl_netisr_work: counter too big (%d)", counter));
1450        NETISR_RUNLOCK(&tracker);
1451        error = SYSCTL_OUT(req, snw_array, sizeof(*snw_array) * counter);
1452        free(snw_array, M_TEMP);
1453        return (error);
1454}
1455
1456SYSCTL_PROC(_net_isr, OID_AUTO, work,
1457    CTLFLAG_RD|CTLTYPE_STRUCT|CTLFLAG_MPSAFE, 0, 0, sysctl_netisr_work,
1458    "S,sysctl_netisr_work",
1459    "Return list of per-workstream, per-protocol work in netisr");
1460
1461#ifdef DDB
1462DB_SHOW_COMMAND(netisr, db_show_netisr)
1463{
1464        struct netisr_workstream *nwsp;
1465        struct netisr_work *nwp;
1466        int first, proto;
1467        u_int cpuid;
1468
1469        db_printf("%3s %6s %5s %5s %5s %8s %8s %8s %8s\n", "CPU", "Proto",
1470            "Len", "WMark", "Max", "Disp", "HDisp", "Drop", "Queue");
1471        CPU_FOREACH(cpuid) {
1472                nwsp = DPCPU_ID_PTR(cpuid, nws);
1473                if (nwsp->nws_intr_event == NULL)
1474                        continue;
1475                first = 1;
1476                for (proto = 0; proto < NETISR_MAXPROT; proto++) {
1477                        if (netisr_proto[proto].np_handler == NULL)
1478                                continue;
1479                        nwp = &nwsp->nws_work[proto];
1480                        if (first) {
1481                                db_printf("%3d ", cpuid);
1482                                first = 0;
1483                        } else
1484                                db_printf("%3s ", "");
1485                        db_printf(
1486                            "%6s %5d %5d %5d %8ju %8ju %8ju %8ju\n",
1487                            netisr_proto[proto].np_name, nwp->nw_len,
1488                            nwp->nw_watermark, nwp->nw_qlimit,
1489                            nwp->nw_dispatched, nwp->nw_hybrid_dispatched,
1490                            nwp->nw_qdrops, nwp->nw_queued);
1491                }
1492        }
1493}
1494#endif
Note: See TracBrowser for help on using the repository browser.