李鸣雷-Raymond LI Raymond's Workspace(李鸣雷的空间)

A R/W deadlock of aggregation in GLD code

I met this problem when running heavy traffic over an aggregation and add/remove interfaces into the aggregation.

This is a good example of read/write deadlock problem.

First, let's explain how the deadlock will happen.

1. When a packet of TCP incoming and triggerres interrupt, it will following below call sequence:     
    driver_xxx_intr -> mac_rx -> (a serie of TCP funcs) -> aggr_m_tx

Above call sequence will acquire rw lock in below sequence:
(1) mac_rx -> mi_rx_lock (as RW_READER) mac.c, LINE 1145
(2) aggr_m_tx -> lg_lock (as RW_READER) aggr_send.c, LINE 220

/* See below code */
http://cvs.opensolaris.org/source/xref/on/usr/src/uts/common/io/mac/mac.c , line 1136 - 1167

void
mac_rx(mac_t *mp, mac_resource_handle_t mrh, mblk_t *bp)
{
	mac_impl_t	*mip = mp->m_impl;
	mac_rx_fn_t	*mrfp;

	/*
	 * Call all registered receive functions.
	 */
	rw_enter(&mip->mi_rx_lock, RW_READER);
	mrfp = mip->mi_mrfp;
	if (mrfp == NULL) {
		/* There are no registered receive functions. */
		freemsgchain(bp);
		rw_exit(&mip->mi_rx_lock);
		return;
	}
	do {
		mblk_t *recv_bp;

		if (mrfp->mrf_nextp != NULL) {
			/* XXX Do we bump a counter if copymsgchain() fails? */
			recv_bp = copymsgchain(bp);
		} else {
			recv_bp = bp;
		}
		if (recv_bp != NULL)
			mrfp->mrf_fn(mrfp->mrf_arg, mrh, recv_bp);
		mrfp = mrfp->mrf_nextp;
	} while (mrfp != NULL);
	rw_exit(&mip->mi_rx_lock);
}

When packet arrived the interface, interrupt handler will call into mac_rx.
In Line 1145,  mip->mi_rx_lock will be acquired as RW_READER here.

While in aggr codes, http://cvs.opensolaris.org/source/xref/on/usr/src/uts/common/io/aggr/aggr_send.c. In LINE 220, it tries to acquire grp->lg_lock as RW_READER.

mblk_t *
aggr_m_tx(void *arg, mblk_t *mp)
{
	aggr_grp_t *grp = arg;
	aggr_port_t *port;
	mblk_t *nextp;
	const mac_txinfo_t *mtp;

	rw_enter(&grp->lg_lock, RW_READER);

	if (grp->lg_ntx_ports == 0) {
		/*
		 * We could have returned from aggr_m_start() before
		 * the ports were actually attached. Drop the chain.
		 */
		rw_exit(&grp->lg_lock);

		freemsgchain(mp);
		return (NULL);
	}

	for (;;) {
		nextp = mp->b_next;
		mp->b_next = NULL;

		port = grp->lg_tx_ports[aggr_send_port(grp, mp)];
		ASSERT(port->lp_state == AGGR_PORT_STATE_ATTACHED);

		rw_exit(&grp->lg_lock);

		/*
		 * We store the transmit info pointer locally in case it
		 * changes between loading mt_fn and mt_arg.
		 */
		mtp = port->lp_txinfo;
		if ((mp = mtp->mt_fn(mtp->mt_arg, mp)) != NULL) {
			mp->b_next = nextp;
			goto done;
		}

		if ((mp = nextp) == NULL)
			goto done;

		rw_enter(&grp->lg_lock, RW_READER);
	}

done:
	return (mp);
}

2. When adminstrator using dladm is to remove an interface from current aggregation(with dladm remove-aggr), it will follow below call sequence:
     aggr_ioctl -> aggr_ioc_remove -> aggr_grp_rem_ports -> aggr_grp_rem_port -> aggr_port_delete -> mac_rx_remove

So the mi_rx_lock and lg_lock will be held in sequence:
(1) aggr_grp_rem_ports -> acquire "lg_lock" (as RW_WRITER), aggr_grp.c LINE 861
(2) mac_rx_remove -> acquire "mi_rx_lock" (as RW_WRITER), mac.c LINE 941

/* See below code */
http://cvs.opensolaris.org/source/xref/on/usr/src/uts/common/io/aggr/aggr_grp.c,  LINE 842 - 909,

int
aggr_grp_rem_ports(uint32_t key, uint_t nports, laioc_port_t *ports)
{
	int rc = 0, i;
	aggr_grp_t *grp = NULL;
	aggr_port_t *port;
	boolean_t notify = B_FALSE, grp_mac_addr_changed;

	/* get group corresponding to key */
	rw_enter(&aggr_grp_lock, RW_READER);
	if (mod_hash_find(aggr_grp_hash, GRP_HASH_KEY(key),
	    (mod_hash_val_t *)&grp) != 0) {
		rw_exit(&aggr_grp_lock);
		return (ENOENT);
	}
	AGGR_GRP_REFHOLD(grp);
	rw_exit(&aggr_grp_lock);

	AGGR_LACP_LOCK(grp);
	rw_enter(&grp->lg_lock, RW_WRITER);

	/* we need to keep at least one port per group */
	if (nports >= grp->lg_nports) {
		rc = EINVAL;
		goto bail;
	}

	/* first verify that all the groups are valid */
	for (i = 0; i < nports; i++) {
		if (aggr_grp_port_lookup(grp, ports[i].lp_devname,
		    ports[i].lp_port) == NULL) {
			/* port not found */
			rc = ENOENT;
			goto bail;
		}
	}

	/* remove the specified ports from group */
	for (i = 0; i < nports && !grp->lg_closing; i++) {
		/* lookup port */
		port = aggr_grp_port_lookup(grp, ports[i].lp_devname,
		    ports[i].lp_port);
		ASSERT(port != NULL);

		/* stop port if group has already been started */
		if (grp->lg_started) {
			rw_enter(&port->lp_lock, RW_WRITER);
			aggr_port_stop(port);
			rw_exit(&port->lp_lock);
		}

		/* remove port from group */
		rc = aggr_grp_rem_port(grp, port, &grp_mac_addr_changed);
		ASSERT(rc == 0);
		notify = notify || grp_mac_addr_changed;
	}

bail:
	rw_exit(&grp->lg_lock);
	AGGR_LACP_UNLOCK(grp);
	if (notify && !grp->lg_closing)
		mac_unicst_update(&grp->lg_mac, grp->lg_addr);
	if (rc == 0 && !grp->lg_closing)
		mac_resource_update(&grp->lg_mac);
	AGGR_GRP_REFRELE(grp);

	return (rc);
}

http://cvs.opensolaris.org/source/xref/on/usr/src/uts/common/io/mac/mac.c, LINE 930 - 953,

void
mac_rx_remove(mac_handle_t mh, mac_rx_handle_t mrh)
{
	mac_impl_t		*mip = (mac_impl_t *)mh;
	mac_rx_fn_t		*mrfp = (mac_rx_fn_t *)mrh;
	mac_rx_fn_t		**pp;
	mac_rx_fn_t		*p;

	/*
	 * Search the 'rx' callback list for the function closure.
	 */
	rw_enter(&(mip->mi_rx_lock), RW_WRITER);
	for (pp = &(mip->mi_mrfp); (p = *pp) != NULL; pp = &(p->mrf_nextp)) {
		if (p == mrfp)
			break;
	}
	ASSERT(p != NULL);

	/* Remove it from the list. */
	*pp = p->mrf_nextp;
	kmem_free(mrfp, sizeof (mac_rx_fn_t));
	rw_exit(&(mip->mi_rx_lock));
}

3. The deadlock will happen in below scene:
(1) thread 1, dladm calls into "aggr_grp_rem_ports" and acquire "lg_lock" as RW_WRITER
(2) thread 2, an packet arrived at the aggregation and interrupt handler calls "mac_rx" and acquire "mi_rx_lock" as RW_READER
(3) thread 2, mac_rx calls into "aggr_m_tx" and tries to acquire "lg_lock" as RW_READER, but currently it is held by step (1) as RW_WRITER, so thread 2 will block
(4) thread 1, aggr_grp_rem_ports calls into "mac_rx_remove" and tris to acquire "mi_rx_lock" as RW_WRITER, but currently it is held by step (2) as RW_READER, so thread 1 will block
(5) The deadload happens

Technorati Tags: OpenSolaris

Posted at 04:35下午 四月 12, 2006 by raymond in Sun  |  评论[0]