The root cause of all of my problems seems to be that I went from C to XDR instead of trying my own .x file. For example, here we see st_node is NULL:

(gdb) p *objp
$3 = {si_op = SPE_OP_EQUAL, si_parens = 0, si_children = 2, si_branches = {{
      st_is_interior = 0, st_node = 0x0}, {st_is_interior = 0, st_node = 0x0}}}

It shouldn't be. I was confused at first, because we are filling it in, so the contents should be zeroed because of the memory allocations. But once we start to use st_node, we should have memory.

The issue is that I had a pointer and I assumed that XDR was allocating the memory for me. Silly boy...

Compare what I had in place:

bool_t
xdr_spe_thunk_t(XDR *xdrs, spe_thunk_t *objp)
{
        if (!xdr_bool(xdrs, &objp->st_is_interior))
                return (FALSE);
        if (objp->st_is_interior) {
                if (!xdr_spe_interior_t(xdrs,
                    (spe_interior_t *)objp->st_node)) {
                        printf("Interior failed\n");
                        return (FALSE);
                }
        } else {
...

with what works:

bool_t
xdr_spe_thunk_t(XDR *xdrs, spe_thunk_t *objp)
{
        if (!xdr_bool(xdrs, &objp->st_is_interior))
                return (FALSE);
        if (objp->st_is_interior) {
                if (xdrs->x_op == XDR_DECODE) {
                        objp->st_u.interior = (spe_interior_t *)
                            calloc(1, sizeof (spe_interior_t));
                        if (!xdr_spe_interior_t(xdrs,
                            objp->st_u.interior)) {
                                printf("Interior failed\n");
                                return (FALSE);
                        }
                } else if (xdrs->x_op == XDR_ENCODE) {
                        if (!xdr_spe_interior_t(xdrs,
                            objp->st_u.interior)) {
                                printf("Interior failed\n");
                                return (FALSE);
                        }
                } else {
                        if (!xdr_spe_interior_t(xdrs,
                            objp->st_u.interior)) {
                                printf("Interior failed\n");
                                return (FALSE);
                        }

                        mem_free(objp->st_u.interior,
                            sizeof (spe_interior_t));
                }
        } else {
...

When we DECODE, we need to make sure we have space.

Next time, I'll generate the .x file and make sure I understand everything that rpcgen complains about. :->

So I realized I was still making things too hard in my debugging efforts. I'm fighting the XDR encode/decode and not the syscall(). I.e., instead of reinstalling the nfsd kernel module, rebooting the server, and using MDB to debug, why don't I have a standalone I can use gdb on?

So I did that and I'm dumping in the same place:

[0]> $c
kmdb_enter+0xb()
debug_enter+0x37(fffffffffb937680)
panicsys+0x3fd(fffffffffb930b08, ffffff0007ced268, fffffffffbc5f700, 1)
vpanic+0x15d()
panic+0x9c()
die+0xd2(e, ffffff0007ced460, 0, 0)
trap+0x160f(ffffff0007ced460, 0, 0)
0xfffffffffb8002c0()
xdr_bool+0x5a(ffffff0007ced810, 0)
nfs`xdr_spe_leaf_t+0x25(ffffff0007ced810, 0)
nfs`xdr_spe_thunk_t+0x3d(ffffff0007ced810, ffffff01e46417f8)
nfs`xdr_vector+0x4f(ffffff0007ced810, ffffff01e46417f8, 2, 10, fffffffff804da40)
nfs`xdr_spe_interior_t+0x7b(ffffff0007ced810, ffffff01e46417e8)
xdr_reference+0x7f(ffffff0007ced810, ffffff01d4a7f020, 30, fffffffff804db20)
xdr_pointer+0x69(ffffff0007ced810, ffffff01d4a7f020, 30, fffffffff804db20)
nfs`xdr_spe_policy_t+0x91(ffffff0007ced810, ffffff01d4a7f010)
nfs`nfs4_sped_svc+0x2ca(8047c10)
nfs`nfssys+0x68(1d, 8047c10)
sys_syscall32+0x1fc()
[0]> ffffff01e46417f8::print spe_thunk_t
{
    st_is_interior = 0
    st_node = 0
}

versus

(gdb) bt
#0  0xfef2a394 in xdr_bool () from /lib/libnsl.so.1
#1  0x080553fe in xdr_spe_leaf_t (xdrs=0x80478c0, objp=0x0) at s.c:2361
#2  0x080553be in xdr_spe_thunk_t (xdrs=0x80478c0, objp=0x8067c04) at s.c:2349
#3  0xfef2aa65 in xdr_vector () from /lib/libnsl.so.1
#4  0x08055511 in xdr_spe_interior_t (xdrs=0x80478c0, objp=0x8067bf8) at s.c:2387
#5  0xfef2be99 in xdr_reference () from /lib/libnsl.so.1
#6  0xfef2bf1b in xdr_pointer () from /lib/libnsl.so.1
#7  0x080555d9 in xdr_spe_policy_t (xdrs=0x80478c0, objp=0x8047850) at s.c:2410
#8  0x08055ba6 in sped_populate (p=0x8067a78) at s.c:2586
#9  0x08055c68 in main (argc=1, argv=0x8047940) at s.c:2606
(gdb) f 1
#1  0x080553fe in xdr_spe_leaf_t (xdrs=0x80478c0, objp=0x0) at s.c:2361
2361            if (!xdr_bool(xdrs, &objp->sl_is_attr)) {
(gdb) f 2
#2  0x080553be in xdr_spe_thunk_t (xdrs=0x80478c0, objp=0x8067c04) at s.c:2349
2349                    if (!xdr_spe_leaf_t(xdrs,
(gdb) p *objp
$1 = {st_is_interior = 0, st_node = 0x0}
(gdb) up
#3  0xfef2aa65 in xdr_vector () from /lib/libnsl.so.1
(gdb) up
#4  0x08055511 in xdr_spe_interior_t (xdrs=0x80478c0, objp=0x8067bf8) at s.c:2387
2387            if (!xdr_vector(xdrs, (char *)objp->si_branches,
(gdb) p objp
$2 = (spe_interior_t *) 0x8067bf8
(gdb) p *objp
$3 = {si_op = SPE_OP_EQUAL, si_parens = 0, si_children = 2, si_branches = {{
      st_is_interior = 0, st_node = 0x0}, {st_is_interior = 0, st_node = 0x0}}}

Ahhh!