[pve-devel] [PATCH zfsonlinux 3/6] cherry-pick 2 patches planned for zfs-0.7.12

Stoiko Ivanov s.ivanov at proxmox.com
Tue Oct 30 11:14:26 CET 2018


both patches have been backported and approved upstream, and will end up in
0.7.12.

Signed-off-by: Stoiko Ivanov <s.ivanov at proxmox.com>
---
 .../0005-Fix-race-in-dnode_check_slots_free.patch  | 256 ++++++
 ...taskq-and-context-switch-cost-of-zio-pipe.patch | 861 +++++++++++++++++++++
 zfs-patches/series                                 |   2 +
 3 files changed, 1119 insertions(+)
 create mode 100644 zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch
 create mode 100644 zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch

diff --git a/zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch b/zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch
new file mode 100644
index 0000000..9cebd00
--- /dev/null
+++ b/zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch
@@ -0,0 +1,256 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Tom Caputi <tcaputi at datto.com>
+Date: Tue, 10 Apr 2018 14:15:05 -0400
+Subject: [PATCH] Fix race in dnode_check_slots_free()
+
+Currently, dnode_check_slots_free() works by checking dn->dn_type
+in the dnode to determine if the dnode is reclaimable. However,
+there is a small window of time between dnode_free_sync() in the
+first call to dsl_dataset_sync() and when the useraccounting code
+is run when the type is set DMU_OT_NONE, but the dnode is not yet
+evictable, leading to crashes. This patch adds the ability for
+dnodes to track which txg they were last dirtied in and adds a
+check for this before performing the reclaim.
+
+This patch also corrects several instances when dn_dirty_link was
+treated as a list_node_t when it is technically a multilist_node_t.
+
+Reviewed-by: Brian Behlendorf <behlendorf1 at llnl.gov>
+Signed-off-by: Tom Caputi <tcaputi at datto.com>
+Requires-spl: spl-0.7-release
+Issue #7147
+Issue #7388
+Issue #7997
+
+(cherry-picked from behlendorf/issue-7997 4764f6f3be90be073d2700653dff286371e52583)
+Signed-off-by: Stoiko Ivanov <s.ivanov at proxmox.com>
+---
+ include/sys/dmu_impl.h  |  1 +
+ include/sys/dnode.h     |  4 ++++
+ module/zfs/dbuf.c       |  3 +++
+ module/zfs/dmu.c        |  2 +-
+ module/zfs/dmu_objset.c | 15 +++++++++++++++
+ module/zfs/dnode.c      | 29 +++++++++++++++++++----------
+ 6 files changed, 43 insertions(+), 11 deletions(-)
+
+diff --git a/include/sys/dmu_impl.h b/include/sys/dmu_impl.h
+index 65e417e3..03a63077 100644
+--- a/include/sys/dmu_impl.h
++++ b/include/sys/dmu_impl.h
+@@ -161,6 +161,7 @@ extern "C" {
+  * 	dn_allocated_txg
+  * 	dn_free_txg
+  * 	dn_assigned_txg
++ * 	dn_dirty_txg
+  * 	dd_assigned_tx
+  * 	dn_notxholds
+  * 	dn_dirtyctx
+diff --git a/include/sys/dnode.h b/include/sys/dnode.h
+index ea7defe1..2dd087b3 100644
+--- a/include/sys/dnode.h
++++ b/include/sys/dnode.h
+@@ -260,6 +260,7 @@ struct dnode {
+ 	uint64_t dn_allocated_txg;
+ 	uint64_t dn_free_txg;
+ 	uint64_t dn_assigned_txg;
++	uint64_t dn_dirty_txg;			/* txg dnode was last dirtied */
+ 	kcondvar_t dn_notxholds;
+ 	enum dnode_dirtycontext dn_dirtyctx;
+ 	uint8_t *dn_dirtyctx_firstset;		/* dbg: contents meaningless */
+@@ -362,6 +363,9 @@ void dnode_evict_dbufs(dnode_t *dn);
+ void dnode_evict_bonus(dnode_t *dn);
+ void dnode_free_interior_slots(dnode_t *dn);
+ 
++#define	DNODE_IS_DIRTY(_dn)						\
++	((_dn)->dn_dirty_txg >= spa_syncing_txg((_dn)->dn_objset->os_spa))
++
+ #define	DNODE_IS_CACHEABLE(_dn)						\
+ 	((_dn)->dn_objset->os_primary_cache == ZFS_CACHE_ALL ||		\
+ 	(DMU_OT_IS_METADATA((_dn)->dn_type) &&				\
+diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c
+index 4ee121f5..6edb39d6 100644
+--- a/module/zfs/dbuf.c
++++ b/module/zfs/dbuf.c
+@@ -1606,6 +1606,9 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
+ 			    FTAG);
+ 		}
+ 	}
++
++	if (tx->tx_txg > dn->dn_dirty_txg)
++		dn->dn_dirty_txg = tx->tx_txg;
+ 	mutex_exit(&dn->dn_mtx);
+ 
+ 	if (db->db_blkid == DMU_SPILL_BLKID)
+diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c
+index 6f09aa2f..a09ac4f9 100644
+--- a/module/zfs/dmu.c
++++ b/module/zfs/dmu.c
+@@ -2044,7 +2044,7 @@ dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
+ 	 * Check if dnode is dirty
+ 	 */
+ 	for (i = 0; i < TXG_SIZE; i++) {
+-		if (list_link_active(&dn->dn_dirty_link[i])) {
++		if (multilist_link_active(&dn->dn_dirty_link[i])) {
+ 			clean = B_FALSE;
+ 			break;
+ 		}
+diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c
+index 449ebedf..0bed2d3e 100644
+--- a/module/zfs/dmu_objset.c
++++ b/module/zfs/dmu_objset.c
+@@ -1213,10 +1213,23 @@ dmu_objset_sync_dnodes(multilist_sublist_t *list, dmu_tx_t *tx)
+ 		ASSERT3U(dn->dn_nlevels, <=, DN_MAX_LEVELS);
+ 		multilist_sublist_remove(list, dn);
+ 
++		/*
++		 * If we are not doing useraccounting (os_synced_dnodes == NULL)
++		 * we are done with this dnode for this txg. Unset dn_dirty_txg
++		 * if later txgs aren't dirtying it so that future holders do
++		 * not get a stale value. Otherwise, we will do this in
++		 * userquota_updates_task() when processing has completely
++		 * finished for this txg.
++		 */
+ 		multilist_t *newlist = dn->dn_objset->os_synced_dnodes;
+ 		if (newlist != NULL) {
+ 			(void) dnode_add_ref(dn, newlist);
+ 			multilist_insert(newlist, dn);
++		} else {
++			mutex_enter(&dn->dn_mtx);
++			if (dn->dn_dirty_txg == tx->tx_txg)
++				dn->dn_dirty_txg = 0;
++			mutex_exit(&dn->dn_mtx);
+ 		}
+ 
+ 		dnode_sync(dn, tx);
+@@ -1621,6 +1634,8 @@ userquota_updates_task(void *arg)
+ 				dn->dn_id_flags |= DN_ID_CHKED_BONUS;
+ 		}
+ 		dn->dn_id_flags &= ~(DN_ID_NEW_EXIST);
++		if (dn->dn_dirty_txg == spa_syncing_txg(os->os_spa))
++			dn->dn_dirty_txg = 0;
+ 		mutex_exit(&dn->dn_mtx);
+ 
+ 		multilist_sublist_remove(list, dn);
+diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c
+index d465b545..4a169c49 100644
+--- a/module/zfs/dnode.c
++++ b/module/zfs/dnode.c
+@@ -137,7 +137,7 @@ dnode_cons(void *arg, void *unused, int kmflag)
+ 	bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz));
+ 
+ 	for (i = 0; i < TXG_SIZE; i++) {
+-		list_link_init(&dn->dn_dirty_link[i]);
++		multilist_link_init(&dn->dn_dirty_link[i]);
+ 		dn->dn_free_ranges[i] = NULL;
+ 		list_create(&dn->dn_dirty_records[i],
+ 		    sizeof (dbuf_dirty_record_t),
+@@ -147,6 +147,7 @@ dnode_cons(void *arg, void *unused, int kmflag)
+ 	dn->dn_allocated_txg = 0;
+ 	dn->dn_free_txg = 0;
+ 	dn->dn_assigned_txg = 0;
++	dn->dn_dirty_txg = 0;
+ 	dn->dn_dirtyctx = 0;
+ 	dn->dn_dirtyctx_firstset = NULL;
+ 	dn->dn_bonus = NULL;
+@@ -184,7 +185,7 @@ dnode_dest(void *arg, void *unused)
+ 	ASSERT(!list_link_active(&dn->dn_link));
+ 
+ 	for (i = 0; i < TXG_SIZE; i++) {
+-		ASSERT(!list_link_active(&dn->dn_dirty_link[i]));
++		ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
+ 		ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
+ 		list_destroy(&dn->dn_dirty_records[i]);
+ 		ASSERT0(dn->dn_next_nblkptr[i]);
+@@ -199,6 +200,7 @@ dnode_dest(void *arg, void *unused)
+ 	ASSERT0(dn->dn_allocated_txg);
+ 	ASSERT0(dn->dn_free_txg);
+ 	ASSERT0(dn->dn_assigned_txg);
++	ASSERT0(dn->dn_dirty_txg);
+ 	ASSERT0(dn->dn_dirtyctx);
+ 	ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
+ 	ASSERT3P(dn->dn_bonus, ==, NULL);
+@@ -523,6 +525,7 @@ dnode_destroy(dnode_t *dn)
+ 	dn->dn_allocated_txg = 0;
+ 	dn->dn_free_txg = 0;
+ 	dn->dn_assigned_txg = 0;
++	dn->dn_dirty_txg = 0;
+ 
+ 	dn->dn_dirtyctx = 0;
+ 	if (dn->dn_dirtyctx_firstset != NULL) {
+@@ -592,6 +595,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
+ 	ASSERT0(dn->dn_maxblkid);
+ 	ASSERT0(dn->dn_allocated_txg);
+ 	ASSERT0(dn->dn_assigned_txg);
++	ASSERT0(dn->dn_dirty_txg);
+ 	ASSERT(refcount_is_zero(&dn->dn_tx_holds));
+ 	ASSERT3U(refcount_count(&dn->dn_holds), <=, 1);
+ 	ASSERT(avl_is_empty(&dn->dn_dbufs));
+@@ -604,7 +608,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
+ 		ASSERT0(dn->dn_next_bonustype[i]);
+ 		ASSERT0(dn->dn_rm_spillblk[i]);
+ 		ASSERT0(dn->dn_next_blksz[i]);
+-		ASSERT(!list_link_active(&dn->dn_dirty_link[i]));
++		ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
+ 		ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL);
+ 		ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
+ 	}
+@@ -779,6 +783,7 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
+ 	ndn->dn_allocated_txg = odn->dn_allocated_txg;
+ 	ndn->dn_free_txg = odn->dn_free_txg;
+ 	ndn->dn_assigned_txg = odn->dn_assigned_txg;
++	ndn->dn_dirty_txg = odn->dn_dirty_txg;
+ 	ndn->dn_dirtyctx = odn->dn_dirtyctx;
+ 	ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
+ 	ASSERT(refcount_count(&odn->dn_tx_holds) == 0);
+@@ -845,6 +850,7 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn)
+ 	odn->dn_allocated_txg = 0;
+ 	odn->dn_free_txg = 0;
+ 	odn->dn_assigned_txg = 0;
++	odn->dn_dirty_txg = 0;
+ 	odn->dn_dirtyctx = 0;
+ 	odn->dn_dirtyctx_firstset = NULL;
+ 	odn->dn_have_spill = B_FALSE;
+@@ -1069,6 +1075,10 @@ dnode_check_slots_free(dnode_children_t *children, int idx, int slots)
+ {
+ 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
+ 
++	/*
++	 * If all dnode slots are either already free or
++	 * evictable return B_TRUE.
++	 */
+ 	for (int i = idx; i < idx + slots; i++) {
+ 		dnode_handle_t *dnh = &children->dnc_children[i];
+ 		dnode_t *dn = dnh->dnh_dnode;
+@@ -1077,18 +1087,17 @@ dnode_check_slots_free(dnode_children_t *children, int idx, int slots)
+ 			continue;
+ 		} else if (DN_SLOT_IS_PTR(dn)) {
+ 			mutex_enter(&dn->dn_mtx);
+-			dmu_object_type_t type = dn->dn_type;
++			boolean_t can_free = (dn->dn_type == DMU_OT_NONE &&
++			    !DNODE_IS_DIRTY(dn));
+ 			mutex_exit(&dn->dn_mtx);
+ 
+-			if (type != DMU_OT_NONE)
++			if (!can_free)
+ 				return (B_FALSE);
+-
+-			continue;
++			else
++				continue;
+ 		} else {
+ 			return (B_FALSE);
+ 		}
+-
+-		return (B_FALSE);
+ 	}
+ 
+ 	return (B_TRUE);
+@@ -1594,7 +1603,7 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
+ 	/*
+ 	 * If we are already marked dirty, we're done.
+ 	 */
+-	if (list_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) {
++	if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) {
+ 		multilist_sublist_unlock(mls);
+ 		return;
+ 	}
diff --git a/zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch b/zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch
new file mode 100644
index 0000000..92dda45
--- /dev/null
+++ b/zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch
@@ -0,0 +1,861 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Matthew Ahrens <mahrens at delphix.com>
+Date: Thu, 2 Aug 2018 15:51:45 -0700
+Subject: [PATCH] Reduce taskq and context-switch cost of zio pipe
+
+When doing a read from disk, ZFS creates 3 ZIO's: a zio_null(), the
+logical zio_read(), and then a physical zio. Currently, each of these
+results in a separate taskq_dispatch(zio_execute).
+
+On high-read-iops workloads, this causes a significant performance
+impact. By processing all 3 ZIO's in a single taskq entry, we reduce the
+overhead on taskq locking and context switching.  We accomplish this by
+allowing zio_done() to return a "next zio to execute" to zio_execute().
+
+This results in a ~12% performance increase for random reads, from
+96,000 iops to 108,000 iops (with recordsize=8k, on SSD's).
+
+Reviewed by: Pavel Zakharov <pavel.zakharov at delphix.com>
+Reviewed-by: Brian Behlendorf <behlendorf1 at llnl.gov>
+Reviewed by: George Wilson <george.wilson at delphix.com>
+Signed-off-by: Matthew Ahrens <mahrens at delphix.com>
+External-issue: DLPX-59292
+Requires-spl: spl-0.7-release
+Closes #7736
+
+(cherry-picked from behlendorf/issue-7736 496657ab3bcfeb638b1786e1759980ccfcacb08e)
+Signed-off-by: Stoiko Ivanov <s.ivanov at proxmox.com>
+---
+ include/sys/zio.h |   4 +-
+ module/zfs/zio.c  | 250 +++++++++++++++++++++++++++++-------------------------
+ 2 files changed, 137 insertions(+), 117 deletions(-)
+
+diff --git a/include/sys/zio.h b/include/sys/zio.h
+index 4b0eecc2..3618912c 100644
+--- a/include/sys/zio.h
++++ b/include/sys/zio.h
+@@ -237,7 +237,7 @@ enum zio_child {
+ #define	ZIO_CHILD_DDT_BIT		ZIO_CHILD_BIT(ZIO_CHILD_DDT)
+ #define	ZIO_CHILD_LOGICAL_BIT		ZIO_CHILD_BIT(ZIO_CHILD_LOGICAL)
+ #define	ZIO_CHILD_ALL_BITS					\
+-	(ZIO_CHILD_VDEV_BIT | ZIO_CHILD_GANG_BIT | 		\
++	(ZIO_CHILD_VDEV_BIT | ZIO_CHILD_GANG_BIT |		\
+ 	ZIO_CHILD_DDT_BIT | ZIO_CHILD_LOGICAL_BIT)
+ 
+ enum zio_wait_type {
+@@ -375,7 +375,7 @@ typedef struct zio_transform {
+ 	struct zio_transform	*zt_next;
+ } zio_transform_t;
+ 
+-typedef int zio_pipe_stage_t(zio_t *zio);
++typedef zio_t *zio_pipe_stage_t(zio_t *zio);
+ 
+ /*
+  * The io_reexecute flags are distinct from io_flags because the child must
+diff --git a/module/zfs/zio.c b/module/zfs/zio.c
+index 9a465e1b..b08b4747 100644
+--- a/module/zfs/zio.c
++++ b/module/zfs/zio.c
+@@ -75,9 +75,6 @@ uint64_t zio_buf_cache_frees[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
+ 
+ int zio_delay_max = ZIO_DELAY_MAX;
+ 
+-#define	ZIO_PIPELINE_CONTINUE		0x100
+-#define	ZIO_PIPELINE_STOP		0x101
+-
+ #define	BP_SPANB(indblkshift, level) \
+ 	(((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
+ #define	COMPARE_META_LEVEL	0x80000000ul
+@@ -516,7 +513,8 @@ zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
+ 
+ __attribute__((always_inline))
+ static inline void
+-zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
++zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait,
++    zio_t **next_to_executep)
+ {
+ 	uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
+ 	int *errorp = &pio->io_child_error[zio->io_child_type];
+@@ -535,13 +533,33 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
+ 		    ZIO_TASKQ_INTERRUPT;
+ 		pio->io_stall = NULL;
+ 		mutex_exit(&pio->io_lock);
++
+ 		/*
+-		 * Dispatch the parent zio in its own taskq so that
+-		 * the child can continue to make progress. This also
+-		 * prevents overflowing the stack when we have deeply nested
+-		 * parent-child relationships.
++		 * If we can tell the caller to execute this parent next, do
++		 * so.  Otherwise dispatch the parent zio as its own task.
++		 *
++		 * Having the caller execute the parent when possible reduces
++		 * locking on the zio taskq's, reduces context switch
++		 * overhead, and has no recursion penalty.  Note that one
++		 * read from disk typically causes at least 3 zio's: a
++		 * zio_null(), the logical zio_read(), and then a physical
++		 * zio.  When the physical ZIO completes, we are able to call
++		 * zio_done() on all 3 of these zio's from one invocation of
++		 * zio_execute() by returning the parent back to
++		 * zio_execute().  Since the parent isn't executed until this
++		 * thread returns back to zio_execute(), the caller should do
++		 * so promptly.
++		 *
++		 * In other cases, dispatching the parent prevents
++		 * overflowing the stack when we have deeply nested
++		 * parent-child relationships, as we do with the "mega zio"
++		 * of writes for spa_sync(), and the chain of ZIL blocks.
+ 		 */
+-		zio_taskq_dispatch(pio, type, B_FALSE);
++		if (next_to_executep != NULL && *next_to_executep == NULL) {
++			*next_to_executep = pio;
++		} else {
++			zio_taskq_dispatch(pio, type, B_FALSE);
++		}
+ 	} else {
+ 		mutex_exit(&pio->io_lock);
+ 	}
+@@ -1187,7 +1205,7 @@ zio_shrink(zio_t *zio, uint64_t size)
+  * ==========================================================================
+  */
+ 
+-static int
++static zio_t *
+ zio_read_bp_init(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -1221,15 +1239,15 @@ zio_read_bp_init(zio_t *zio)
+ 	if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
+ 		zio->io_pipeline = ZIO_DDT_READ_PIPELINE;
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_write_bp_init(zio_t *zio)
+ {
+ 
+ 	if (!IO_IS_ALLOCATING(zio))
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 
+ 	ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
+ 
+@@ -1244,7 +1262,7 @@ zio_write_bp_init(zio_t *zio)
+ 		zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
+ 
+ 		if (BP_IS_EMBEDDED(bp))
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 
+ 		/*
+ 		 * If we've been overridden and nopwrite is set then
+@@ -1255,13 +1273,13 @@ zio_write_bp_init(zio_t *zio)
+ 			ASSERT(!zp->zp_dedup);
+ 			ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum);
+ 			zio->io_flags |= ZIO_FLAG_NOPWRITE;
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 		}
+ 
+ 		ASSERT(!zp->zp_nopwrite);
+ 
+ 		if (BP_IS_HOLE(bp) || !zp->zp_dedup)
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 
+ 		ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &
+ 		    ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify);
+@@ -1269,7 +1287,7 @@ zio_write_bp_init(zio_t *zio)
+ 		if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) {
+ 			BP_SET_DEDUP(bp, 1);
+ 			zio->io_pipeline |= ZIO_STAGE_DDT_WRITE;
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 		}
+ 
+ 		/*
+@@ -1281,10 +1299,10 @@ zio_write_bp_init(zio_t *zio)
+ 		zio->io_pipeline = zio->io_orig_pipeline;
+ 	}
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_write_compress(zio_t *zio)
+ {
+ 	spa_t *spa = zio->io_spa;
+@@ -1303,11 +1321,11 @@ zio_write_compress(zio_t *zio)
+ 	 */
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT |
+ 	    ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	if (!IO_IS_ALLOCATING(zio))
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 
+ 	if (zio->io_children_ready != NULL) {
+ 		/*
+@@ -1366,7 +1384,7 @@ zio_write_compress(zio_t *zio)
+ 			zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
+ 			ASSERT(spa_feature_is_active(spa,
+ 			    SPA_FEATURE_EMBEDDED_DATA));
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 		} else {
+ 			/*
+ 			 * Round up compressed size up to the ashift
+@@ -1459,10 +1477,10 @@ zio_write_compress(zio_t *zio)
+ 			zio->io_pipeline |= ZIO_STAGE_NOP_WRITE;
+ 		}
+ 	}
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_free_bp_init(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -1472,7 +1490,7 @@ zio_free_bp_init(zio_t *zio)
+ 			zio->io_pipeline = ZIO_DDT_FREE_PIPELINE;
+ 	}
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -1541,12 +1559,12 @@ zio_taskq_member(zio_t *zio, zio_taskq_type_t q)
+ 	return (B_FALSE);
+ }
+ 
+-static int
++static zio_t *
+ zio_issue_async(zio_t *zio)
+ {
+ 	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
+ 
+-	return (ZIO_PIPELINE_STOP);
++	return (NULL);
+ }
+ 
+ void
+@@ -1687,14 +1705,13 @@ __attribute__((always_inline))
+ static inline void
+ __zio_execute(zio_t *zio)
+ {
+-	zio->io_executor = curthread;
+-
+ 	ASSERT3U(zio->io_queued_timestamp, >, 0);
+ 
+ 	while (zio->io_stage < ZIO_STAGE_DONE) {
+ 		enum zio_stage pipeline = zio->io_pipeline;
+ 		enum zio_stage stage = zio->io_stage;
+-		int rv;
++
++		zio->io_executor = curthread;
+ 
+ 		ASSERT(!MUTEX_HELD(&zio->io_lock));
+ 		ASSERT(ISP2(stage));
+@@ -1736,12 +1753,16 @@ __zio_execute(zio_t *zio)
+ 
+ 		zio->io_stage = stage;
+ 		zio->io_pipeline_trace |= zio->io_stage;
+-		rv = zio_pipeline[highbit64(stage) - 1](zio);
+ 
+-		if (rv == ZIO_PIPELINE_STOP)
+-			return;
++		/*
++		 * The zio pipeline stage returns the next zio to execute
++		 * (typically the same as this one), or NULL if we should
++		 * stop.
++		 */
++		zio = zio_pipeline[highbit64(stage) - 1](zio);
+ 
+-		ASSERT(rv == ZIO_PIPELINE_CONTINUE);
++		if (zio == NULL)
++			return;
+ 	}
+ }
+ 
+@@ -2215,7 +2236,7 @@ zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, abd_t *data,
+ 		zio_nowait(zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_gang_assemble(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -2227,16 +2248,16 @@ zio_gang_assemble(zio_t *zio)
+ 
+ 	zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_gang_issue(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+ 
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);
+@@ -2250,7 +2271,7 @@ zio_gang_issue(zio_t *zio)
+ 
+ 	zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ static void
+@@ -2290,7 +2311,7 @@ zio_write_gang_done(zio_t *zio)
+ 	abd_put(zio->io_abd);
+ }
+ 
+-static int
++static zio_t *
+ zio_write_gang_block(zio_t *pio)
+ {
+ 	spa_t *spa = pio->io_spa;
+@@ -2349,7 +2370,7 @@ zio_write_gang_block(zio_t *pio)
+ 		}
+ 
+ 		pio->io_error = error;
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (pio);
+ 	}
+ 
+ 	if (pio == gio) {
+@@ -2423,7 +2444,7 @@ zio_write_gang_block(zio_t *pio)
+ 
+ 	zio_nowait(zio);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (pio);
+ }
+ 
+ /*
+@@ -2444,7 +2465,7 @@ zio_write_gang_block(zio_t *pio)
+  * used for nopwrite, assuming that the salt and the checksums
+  * themselves remain secret.
+  */
+-static int
++static zio_t *
+ zio_nop_write(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -2471,7 +2492,7 @@ zio_nop_write(zio_t *zio)
+ 	    BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) ||
+ 	    BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) ||
+ 	    zp->zp_copies != BP_GET_NDVAS(bp_orig))
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 
+ 	/*
+ 	 * If the checksums match then reset the pipeline so that we
+@@ -2491,7 +2512,7 @@ zio_nop_write(zio_t *zio)
+ 		zio->io_flags |= ZIO_FLAG_NOPWRITE;
+ 	}
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -2519,7 +2540,7 @@ zio_ddt_child_read_done(zio_t *zio)
+ 	mutex_exit(&pio->io_lock);
+ }
+ 
+-static int
++static zio_t *
+ zio_ddt_read_start(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -2540,7 +2561,7 @@ zio_ddt_read_start(zio_t *zio)
+ 		zio->io_vsd = dde;
+ 
+ 		if (ddp_self == NULL)
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 
+ 		for (p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
+ 			if (ddp->ddp_phys_birth == 0 || ddp == ddp_self)
+@@ -2553,23 +2574,23 @@ zio_ddt_read_start(zio_t *zio)
+ 			    zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) |
+ 			    ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark));
+ 		}
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 	}
+ 
+ 	zio_nowait(zio_read(zio, zio->io_spa, bp,
+ 	    zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority,
+ 	    ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark));
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_ddt_read_done(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+ 
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	ASSERT(BP_GET_DEDUP(bp));
+@@ -2581,12 +2602,12 @@ zio_ddt_read_done(zio_t *zio)
+ 		ddt_entry_t *dde = zio->io_vsd;
+ 		if (ddt == NULL) {
+ 			ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 		}
+ 		if (dde == NULL) {
+ 			zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1;
+ 			zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
+-			return (ZIO_PIPELINE_STOP);
++			return (NULL);
+ 		}
+ 		if (dde->dde_repair_abd != NULL) {
+ 			abd_copy(zio->io_abd, dde->dde_repair_abd,
+@@ -2599,7 +2620,7 @@ zio_ddt_read_done(zio_t *zio)
+ 
+ 	ASSERT(zio->io_vsd == NULL);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ static boolean_t
+@@ -2780,7 +2801,7 @@ zio_ddt_ditto_write_done(zio_t *zio)
+ 	ddt_exit(ddt);
+ }
+ 
+-static int
++static zio_t *
+ zio_ddt_write(zio_t *zio)
+ {
+ 	spa_t *spa = zio->io_spa;
+@@ -2822,7 +2843,7 @@ zio_ddt_write(zio_t *zio)
+ 		}
+ 		zio->io_pipeline = ZIO_WRITE_PIPELINE;
+ 		ddt_exit(ddt);
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 	}
+ 
+ 	ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
+@@ -2848,7 +2869,7 @@ zio_ddt_write(zio_t *zio)
+ 			zio->io_bp_override = NULL;
+ 			BP_ZERO(bp);
+ 			ddt_exit(ddt);
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 		}
+ 
+ 		dio = zio_write(zio, spa, txg, bp, zio->io_orig_abd,
+@@ -2890,12 +2911,12 @@ zio_ddt_write(zio_t *zio)
+ 	if (dio)
+ 		zio_nowait(dio);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ ddt_entry_t *freedde; /* for debugging */
+ 
+-static int
++static zio_t *
+ zio_ddt_free(zio_t *zio)
+ {
+ 	spa_t *spa = zio->io_spa;
+@@ -2916,7 +2937,7 @@ zio_ddt_free(zio_t *zio)
+ 	}
+ 	ddt_exit(ddt);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -2953,7 +2974,7 @@ zio_io_to_allocate(spa_t *spa)
+ 	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_dva_throttle(zio_t *zio)
+ {
+ 	spa_t *spa = zio->io_spa;
+@@ -2963,7 +2984,7 @@ zio_dva_throttle(zio_t *zio)
+ 	    !spa_normal_class(zio->io_spa)->mc_alloc_throttle_enabled ||
+ 	    zio->io_child_type == ZIO_CHILD_GANG ||
+ 	    zio->io_flags & ZIO_FLAG_NODATA) {
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 	}
+ 
+ 	ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
+@@ -2979,22 +3000,7 @@ zio_dva_throttle(zio_t *zio)
+ 	nio = zio_io_to_allocate(zio->io_spa);
+ 	mutex_exit(&spa->spa_alloc_lock);
+ 
+-	if (nio == zio)
+-		return (ZIO_PIPELINE_CONTINUE);
+-
+-	if (nio != NULL) {
+-		ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE);
+-		/*
+-		 * We are passing control to a new zio so make sure that
+-		 * it is processed by a different thread. We do this to
+-		 * avoid stack overflows that can occur when parents are
+-		 * throttled and children are making progress. We allow
+-		 * it to go to the head of the taskq since it's already
+-		 * been waiting.
+-		 */
+-		zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE);
+-	}
+-	return (ZIO_PIPELINE_STOP);
++	return (nio);
+ }
+ 
+ void
+@@ -3013,7 +3019,7 @@ zio_allocate_dispatch(spa_t *spa)
+ 	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
+ }
+ 
+-static int
++static zio_t *
+ zio_dva_allocate(zio_t *zio)
+ {
+ 	spa_t *spa = zio->io_spa;
+@@ -3054,18 +3060,18 @@ zio_dva_allocate(zio_t *zio)
+ 		zio->io_error = error;
+ 	}
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_dva_free(zio_t *zio)
+ {
+ 	metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_dva_claim(zio_t *zio)
+ {
+ 	int error;
+@@ -3074,7 +3080,7 @@ zio_dva_claim(zio_t *zio)
+ 	if (error)
+ 		zio->io_error = error;
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -3172,7 +3178,7 @@ zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp)
+  * force the underlying vdev layers to call either zio_execute() or
+  * zio_interrupt() to ensure that the pipeline continues with the correct I/O.
+  */
+-static int
++static zio_t *
+ zio_vdev_io_start(zio_t *zio)
+ {
+ 	vdev_t *vd = zio->io_vd;
+@@ -3192,7 +3198,7 @@ zio_vdev_io_start(zio_t *zio)
+ 		 * The mirror_ops handle multiple DVAs in a single BP.
+ 		 */
+ 		vdev_mirror_ops.vdev_op_io_start(zio);
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	ASSERT3P(zio->io_logical, !=, zio);
+@@ -3269,31 +3275,31 @@ zio_vdev_io_start(zio_t *zio)
+ 	    !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {
+ 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);
+ 		zio_vdev_io_bypass(zio);
+-		return (ZIO_PIPELINE_CONTINUE);
++		return (zio);
+ 	}
+ 
+ 	if (vd->vdev_ops->vdev_op_leaf &&
+ 	    (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) {
+ 
+ 		if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio))
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 
+ 		if ((zio = vdev_queue_io(zio)) == NULL)
+-			return (ZIO_PIPELINE_STOP);
++			return (NULL);
+ 
+ 		if (!vdev_accessible(vd, zio)) {
+ 			zio->io_error = SET_ERROR(ENXIO);
+ 			zio_interrupt(zio);
+-			return (ZIO_PIPELINE_STOP);
++			return (NULL);
+ 		}
+ 		zio->io_delay = gethrtime();
+ 	}
+ 
+ 	vd->vdev_ops->vdev_op_io_start(zio);
+-	return (ZIO_PIPELINE_STOP);
++	return (NULL);
+ }
+ 
+-static int
++static zio_t *
+ zio_vdev_io_done(zio_t *zio)
+ {
+ 	vdev_t *vd = zio->io_vd;
+@@ -3301,7 +3307,7 @@ zio_vdev_io_done(zio_t *zio)
+ 	boolean_t unexpected_error = B_FALSE;
+ 
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE);
+@@ -3337,7 +3343,7 @@ zio_vdev_io_done(zio_t *zio)
+ 	if (unexpected_error)
+ 		VERIFY(vdev_probe(vd, zio) == NULL);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -3366,13 +3372,13 @@ zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored)
+ 	zcr->zcr_free = zio_abd_free;
+ }
+ 
+-static int
++static zio_t *
+ zio_vdev_io_assess(zio_t *zio)
+ {
+ 	vdev_t *vd = zio->io_vd;
+ 
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
+@@ -3402,7 +3408,7 @@ zio_vdev_io_assess(zio_t *zio)
+ 		zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1;
+ 		zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE,
+ 		    zio_requeue_io_start_cut_in_line);
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	/*
+@@ -3442,7 +3448,7 @@ zio_vdev_io_assess(zio_t *zio)
+ 		zio->io_physdone(zio->io_logical);
+ 	}
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ void
+@@ -3477,7 +3483,7 @@ zio_vdev_io_bypass(zio_t *zio)
+  * Generate and verify checksums
+  * ==========================================================================
+  */
+-static int
++static zio_t *
+ zio_checksum_generate(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -3491,7 +3497,7 @@ zio_checksum_generate(zio_t *zio)
+ 		checksum = zio->io_prop.zp_checksum;
+ 
+ 		if (checksum == ZIO_CHECKSUM_OFF)
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 
+ 		ASSERT(checksum == ZIO_CHECKSUM_LABEL);
+ 	} else {
+@@ -3505,10 +3511,10 @@ zio_checksum_generate(zio_t *zio)
+ 
+ 	zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+-static int
++static zio_t *
+ zio_checksum_verify(zio_t *zio)
+ {
+ 	zio_bad_cksum_t info;
+@@ -3523,7 +3529,7 @@ zio_checksum_verify(zio_t *zio)
+ 		 * We're either verifying a label checksum, or nothing at all.
+ 		 */
+ 		if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)
+-			return (ZIO_PIPELINE_CONTINUE);
++			return (zio);
+ 
+ 		ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL);
+ 	}
+@@ -3538,7 +3544,7 @@ zio_checksum_verify(zio_t *zio)
+ 		}
+ 	}
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -3581,7 +3587,7 @@ zio_worst_error(int e1, int e2)
+  * I/O completion
+  * ==========================================================================
+  */
+-static int
++static zio_t *
+ zio_ready(zio_t *zio)
+ {
+ 	blkptr_t *bp = zio->io_bp;
+@@ -3590,7 +3596,7 @@ zio_ready(zio_t *zio)
+ 
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT,
+ 	    ZIO_WAIT_READY)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	if (zio->io_ready) {
+@@ -3636,7 +3642,7 @@ zio_ready(zio_t *zio)
+ 	 */
+ 	for (; pio != NULL; pio = pio_next) {
+ 		pio_next = zio_walk_parents(zio, &zl);
+-		zio_notify_parent(pio, zio, ZIO_WAIT_READY);
++		zio_notify_parent(pio, zio, ZIO_WAIT_READY, NULL);
+ 	}
+ 
+ 	if (zio->io_flags & ZIO_FLAG_NODATA) {
+@@ -3652,7 +3658,7 @@ zio_ready(zio_t *zio)
+ 	    zio->io_spa->spa_syncing_txg == zio->io_txg)
+ 		zio_handle_ignored_writes(zio);
+ 
+-	return (ZIO_PIPELINE_CONTINUE);
++	return (zio);
+ }
+ 
+ /*
+@@ -3716,7 +3722,7 @@ zio_dva_throttle_done(zio_t *zio)
+ 	zio_allocate_dispatch(zio->io_spa);
+ }
+ 
+-static int
++static zio_t *
+ zio_done(zio_t *zio)
+ {
+ 	/*
+@@ -3733,7 +3739,7 @@ zio_done(zio_t *zio)
+ 	 * wait for them and then repeat this pipeline stage.
+ 	 */
+ 	if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) {
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	/*
+@@ -3957,7 +3963,12 @@ zio_done(zio_t *zio)
+ 			if ((pio->io_flags & ZIO_FLAG_GODFATHER) &&
+ 			    (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) {
+ 				zio_remove_child(pio, zio, remove_zl);
+-				zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
++				/*
++				 * This is a rare code path, so we don't
++				 * bother with "next_to_execute".
++				 */
++				zio_notify_parent(pio, zio, ZIO_WAIT_DONE,
++				    NULL);
+ 			}
+ 		}
+ 
+@@ -3969,7 +3980,11 @@ zio_done(zio_t *zio)
+ 			 */
+ 			ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
+ 			zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE;
+-			zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
++			/*
++			 * This is a rare code path, so we don't bother with
++			 * "next_to_execute".
++			 */
++			zio_notify_parent(pio, zio, ZIO_WAIT_DONE, NULL);
+ 		} else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) {
+ 			/*
+ 			 * We'd fail again if we reexecuted now, so suspend
+@@ -3987,7 +4002,7 @@ zio_done(zio_t *zio)
+ 			    (task_func_t *)zio_reexecute, zio, 0,
+ 			    &zio->io_tqent);
+ 		}
+-		return (ZIO_PIPELINE_STOP);
++		return (NULL);
+ 	}
+ 
+ 	ASSERT(zio->io_child_count == 0);
+@@ -4023,12 +4038,17 @@ zio_done(zio_t *zio)
+ 	zio->io_state[ZIO_WAIT_DONE] = 1;
+ 	mutex_exit(&zio->io_lock);
+ 
++	/*
++	 * We are done executing this zio.  We may want to execute a parent
++	 * next.  See the comment in zio_notify_parent().
++	 */
++	zio_t *next_to_execute = NULL;
+ 	zl = NULL;
+ 	for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) {
+ 		zio_link_t *remove_zl = zl;
+ 		pio_next = zio_walk_parents(zio, &zl);
+ 		zio_remove_child(pio, zio, remove_zl);
+-		zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
++		zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next_to_execute);
+ 	}
+ 
+ 	if (zio->io_waiter != NULL) {
+@@ -4040,7 +4060,7 @@ zio_done(zio_t *zio)
+ 		zio_destroy(zio);
+ 	}
+ 
+-	return (ZIO_PIPELINE_STOP);
++	return (next_to_execute);
+ }
+ 
+ /*
diff --git a/zfs-patches/series b/zfs-patches/series
index 5130acc..3b4626f 100644
--- a/zfs-patches/series
+++ b/zfs-patches/series
@@ -2,3 +2,5 @@
 0002-import-with-d-dev-disk-by-id-in-scan-service.patch
 0003-always-load-ZFS-module-on-boot.patch
 0004-Fix-deadlock-between-zfs-umount-snapentry_expire.patch
+0005-Fix-race-in-dnode_check_slots_free.patch
+0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch
-- 
2.11.0





More information about the pve-devel mailing list