[PATCH net-next v2] net/mlx5: allocate DMA pool structs on the pool's NUMA node

[PATCH net-next v2] net/mlx5: allocate DMA pool structs on the pool's NUMA node

[Seongjun Hong]

The mlx5 DMA pool structs (mlx5_dma_pool, mlx5_dma_pool_page and its
block bitmap, and mlx5_frag_buf_node_pools) are allocated without a
node hint, so they land on the node of whichever CPU happens to create
or fill the pool, while the DMA pages they describe are allocated on
the pool's node.

The pool, page and bitmap are dereferenced on every block allocation
and free. Allocate them on the pool's NUMA node as well, so that all
of a pool's state lives on one node.

These allocations happen on the control path, so the performance gain
is expected to be small; the change is mainly for consistency.

Signed-off-by: Seongjun Hong <hsj0512@snu.ac.kr>
---
Changes in v2:
- fix typo and specify the structs are allocated on the control path
- add a prefix net-next to the subject
- Link to v1: https://lore.kernel.org/r/20261005-net-mlx5-numa-allocate-pool-page-v1-1-01f4283932b4@snu.ac.kr
---
 drivers/net/ethernet/mellanox/mlx5/core/alloc.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
index a92cf545bdaf..dcd281c4691d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
@@ -110,7 +110,7 @@ static struct mlx5_dma_pool *mlx5_dma_pool_create(struct mlx5_core_dev *dev,
 {
 	struct mlx5_dma_pool *pool;
 
-	pool = kzalloc_obj(*pool);
+	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
 	if (!pool)
 		return NULL;
 
@@ -127,19 +127,20 @@ mlx5_dma_pool_page_alloc(struct mlx5_dma_pool *pool)
 {
 	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
 	struct mlx5_dma_pool_page *page;
+	int node = pool->node;
 
-	page = kzalloc_obj(*page);
+	page = kzalloc_node(sizeof(*page), GFP_KERNEL, node);
 	if (!page)
 		goto err_out;
 
 	page->pool = pool;
-	page->bitmap = bitmap_zalloc(blocks_per_page, GFP_KERNEL);
+	page->bitmap = bitmap_zalloc_node(blocks_per_page, GFP_KERNEL, node);
 	if (!page->bitmap)
 		goto err_free_page;
 
 	bitmap_fill(page->bitmap, blocks_per_page);
 	page->buf = mlx5_dma_zalloc_coherent_node(pool->dev, PAGE_SIZE,
-						  &page->dma, pool->node);
+						  &page->dma, node);
 	if (!page->buf)
 		goto err_free_bitmap;
 
@@ -278,7 +279,7 @@ mlx5_frag_buf_node_pools_create(struct mlx5_core_dev *dev, int node)
 {
 	struct mlx5_frag_buf_node_pools *node_pools;
 
-	node_pools = kzalloc_obj(*node_pools);
+	node_pools = kzalloc_node(sizeof(*node_pools), GFP_KERNEL, node);
 	if (!node_pools)
 		return NULL;
 

---
base-commit: 8b4e7209c842d8cb9516f1f5ef0a88aa2d8831a6
change-id: 20261005-net-mlx5-numa-allocate-pool-page-2ee20a04147e

Best regards,
-- 
Seongjun Hong

Re: [PATCH net-next v2] net/mlx5: allocate DMA pool structs on the pool's NUMA node

[Jacob Keller]

On 10/6/2026 7:16 AM, Seongjun Hong wrote:
> The mlx5 DMA pool structs (mlx5_dma_pool, mlx5_dma_pool_page and its
> block bitmap, and mlx5_frag_buf_node_pools) are allocated without a
> node hint, so they land on the node of whichever CPU happens to create
> or fill the pool, while the DMA pages they describe are allocated on
> the pool's node.
> 
> The pool, page and bitmap are dereferenced on every block allocation
> and free. Allocate them on the pool's NUMA node as well, so that all
> of a pool's state lives on one node.
> 
> These allocations happen on the control path, so the performance gain
> is expected to be small; the change is mainly for consistency.
> 
> Signed-off-by: Seongjun Hong <hsj0512@snu.ac.kr>
> ---
> Changes in v2:
> - fix typo and specify the structs are allocated on the control path
> - add a prefix net-next to the subject
> - Link to v1: https://lore.kernel.org/r/20261005-net-mlx5-numa-allocate-pool-page-v1-1-01f4283932b4@snu.ac.kr
> ---
>  drivers/net/ethernet/mellanox/mlx5/core/alloc.c | 11 ++++++-----
>  1 file changed, 6 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
> index a92cf545bdaf..dcd281c4691d 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
> @@ -110,7 +110,7 @@ static struct mlx5_dma_pool *mlx5_dma_pool_create(struct mlx5_core_dev *dev,
>  {
>  	struct mlx5_dma_pool *pool;
>  
> -	pool = kzalloc_obj(*pool);
> +	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);

This removes all the benefits of kzalloc_obj() here...

But I guess the __alloc_objs API doesn'thave the ability to add a node?
It seems like we'd benefit from having support for the node parameter..
but it does seem tricky to add. kzalloc_node_obj() could potentially be
added. Staring at the implementation I have no idea how complicated that
would be.

Still, this only really changes two allocations and they're pretty
obviously correct.

Reviewed-by: Jacob Keller <jacob.e.keller@intel.com>

>  	if (!pool)
>  		return NULL;
>  
> @@ -127,19 +127,20 @@ mlx5_dma_pool_page_alloc(struct mlx5_dma_pool *pool)
>  {
>  	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
>  	struct mlx5_dma_pool_page *page;
> +	int node = pool->node;
>  
> -	page = kzalloc_obj(*page);
> +	page = kzalloc_node(sizeof(*page), GFP_KERNEL, node);
>  	if (!page)
>  		goto err_out;
>  
>  	page->pool = pool;
> -	page->bitmap = bitmap_zalloc(blocks_per_page, GFP_KERNEL);
> +	page->bitmap = bitmap_zalloc_node(blocks_per_page, GFP_KERNEL, node);
>  	if (!page->bitmap)
>  		goto err_free_page;
>  
>  	bitmap_fill(page->bitmap, blocks_per_page);
>  	page->buf = mlx5_dma_zalloc_coherent_node(pool->dev, PAGE_SIZE,
> -						  &page->dma, pool->node);
> +						  &page->dma, node);
>  	if (!page->buf)
>  		goto err_free_bitmap;
>  
> @@ -278,7 +279,7 @@ mlx5_frag_buf_node_pools_create(struct mlx5_core_dev *dev, int node)
>  {
>  	struct mlx5_frag_buf_node_pools *node_pools;
>  
> -	node_pools = kzalloc_obj(*node_pools);
> +	node_pools = kzalloc_node(sizeof(*node_pools), GFP_KERNEL, node);
>  	if (!node_pools)
>  		return NULL;
>  
> 
> ---
> base-commit: 8b4e7209c842d8cb9516f1f5ef0a88aa2d8831a6
> change-id: 20261005-net-mlx5-numa-allocate-pool-page-2ee20a04147e
> 
> Best regards,

Re: [PATCH net-next v2] net/mlx5: allocate DMA pool structs on the pool's NUMA node

[Seongjun Hong]

On Tue, Oct 06, 2026 at 02:57:28PM -0700, Jacob Keller wrote:
> On 10/6/2026 7:16 AM, Seongjun Hong wrote:
> > The mlx5 DMA pool structs (mlx5_dma_pool, mlx5_dma_pool_page and its
> > block bitmap, and mlx5_frag_buf_node_pools) are allocated without a
> > node hint, so they land on the node of whichever CPU happens to create
> > or fill the pool, while the DMA pages they describe are allocated on
> > the pool's node.
> > 
> > The pool, page and bitmap are dereferenced on every block allocation
> > and free. Allocate them on the pool's NUMA node as well, so that all
> > of a pool's state lives on one node.
> > 
> > These allocations happen on the control path, so the performance gain
> > is expected to be small; the change is mainly for consistency.
> > 
> > Signed-off-by: Seongjun Hong <hsj0512@snu.ac.kr>
> > ---
> > Changes in v2:
> > - fix typo and specify the structs are allocated on the control path
> > - add a prefix net-next to the subject
> > - Link to v1: https://lore.kernel.org/r/20261005-net-mlx5-numa-allocate-pool-page-v1-1-01f4283932b4@snu.ac.kr
> > ---
> >  drivers/net/ethernet/mellanox/mlx5/core/alloc.c | 11 ++++++-----
> >  1 file changed, 6 insertions(+), 5 deletions(-)
> > 
> > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
> > index a92cf545bdaf..dcd281c4691d 100644
> > --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
> > +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
> > @@ -110,7 +110,7 @@ static struct mlx5_dma_pool *mlx5_dma_pool_create(struct mlx5_core_dev *dev,
> >  {
> >  	struct mlx5_dma_pool *pool;
> >  
> > -	pool = kzalloc_obj(*pool);
> > +	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
> 
> This removes all the benefits of kzalloc_obj() here...

Do you mean type checking on compile?

> 
> But I guess the __alloc_objs API doesn'thave the ability to add a node?
> It seems like we'd benefit from having support for the node parameter..
> but it does seem tricky to add. kzalloc_node_obj() could potentially be
> added. Staring at the implementation I have no idea how complicated that
> would be.
> 
> Still, this only really changes two allocations and they're pretty
> obviously correct.
> 
> Reviewed-by: Jacob Keller <jacob.e.keller@intel.com>
> 

Jacob, thanks for the review.
As far as I know, there is no node-aware variant of _obj APIs yet,
so kzalloc_node() was the only option here.

> >  	if (!pool)
> >  		return NULL;
> >  
> > @@ -127,19 +127,20 @@ mlx5_dma_pool_page_alloc(struct mlx5_dma_pool *pool)
> >  {
> >  	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
> >  	struct mlx5_dma_pool_page *page;
> > +	int node = pool->node;
> >  
> > -	page = kzalloc_obj(*page);
> > +	page = kzalloc_node(sizeof(*page), GFP_KERNEL, node);
> >  	if (!page)
> >  		goto err_out;
> >  
> >  	page->pool = pool;
> > -	page->bitmap = bitmap_zalloc(blocks_per_page, GFP_KERNEL);
> > +	page->bitmap = bitmap_zalloc_node(blocks_per_page, GFP_KERNEL, node);
> >  	if (!page->bitmap)
> >  		goto err_free_page;
> >  
> >  	bitmap_fill(page->bitmap, blocks_per_page);
> >  	page->buf = mlx5_dma_zalloc_coherent_node(pool->dev, PAGE_SIZE,
> > -						  &page->dma, pool->node);
> > +						  &page->dma, node);
> >  	if (!page->buf)
> >  		goto err_free_bitmap;
> >  
> > @@ -278,7 +279,7 @@ mlx5_frag_buf_node_pools_create(struct mlx5_core_dev *dev, int node)
> >  {
> >  	struct mlx5_frag_buf_node_pools *node_pools;
> >  
> > -	node_pools = kzalloc_obj(*node_pools);
> > +	node_pools = kzalloc_node(sizeof(*node_pools), GFP_KERNEL, node);
> >  	if (!node_pools)
> >  		return NULL;
> >  
> > 
> > ---
> > base-commit: 8b4e7209c842d8cb9516f1f5ef0a88aa2d8831a6
> > change-id: 20261005-net-mlx5-numa-allocate-pool-page-2ee20a04147e
> > 
> > Best regards,
> 
> 
> 

-- 
Seongjun Hong