Commit a0fedc85 authored by Jens Axboe's avatar Jens Axboe

Merge branch 'irq/for-block' of...

Merge branch 'irq/for-block' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip into for-4.21/block

Pull in the irq affinity commits, that are staged through Thomas's
tree.

* 'irq/for-block' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  genirq/affinity: Add support for allocating interrupt sets
  genirq/affinity: Pass first vector to __irq_build_affinity_masks()
  genirq/affinity: Move two stage affinity spreading into a helper function
  genirq/affinity: Spread IRQs to all available NUMA nodes
parents 9cf2bab6 6da4b3ab
...@@ -1036,6 +1036,13 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, ...@@ -1036,6 +1036,13 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
if (maxvec < minvec) if (maxvec < minvec)
return -ERANGE; return -ERANGE;
/*
* If the caller is passing in sets, we can't support a range of
* vectors. The caller needs to handle that.
*/
if (affd && affd->nr_sets && minvec != maxvec)
return -EINVAL;
if (WARN_ON_ONCE(dev->msi_enabled)) if (WARN_ON_ONCE(dev->msi_enabled))
return -EINVAL; return -EINVAL;
...@@ -1087,6 +1094,13 @@ static int __pci_enable_msix_range(struct pci_dev *dev, ...@@ -1087,6 +1094,13 @@ static int __pci_enable_msix_range(struct pci_dev *dev,
if (maxvec < minvec) if (maxvec < minvec)
return -ERANGE; return -ERANGE;
/*
* If the caller is passing in sets, we can't support a range of
* supported vectors. The caller needs to handle that.
*/
if (affd && affd->nr_sets && minvec != maxvec)
return -EINVAL;
if (WARN_ON_ONCE(dev->msix_enabled)) if (WARN_ON_ONCE(dev->msix_enabled))
return -EINVAL; return -EINVAL;
......
...@@ -247,10 +247,14 @@ struct irq_affinity_notify { ...@@ -247,10 +247,14 @@ struct irq_affinity_notify {
* the MSI(-X) vector space * the MSI(-X) vector space
* @post_vectors: Don't apply affinity to @post_vectors at end of * @post_vectors: Don't apply affinity to @post_vectors at end of
* the MSI(-X) vector space * the MSI(-X) vector space
* @nr_sets: Length of passed in *sets array
* @sets: Number of affinitized sets
*/ */
struct irq_affinity { struct irq_affinity {
int pre_vectors; int pre_vectors;
int post_vectors; int post_vectors;
int nr_sets;
int *sets;
}; };
#if defined(CONFIG_SMP) #if defined(CONFIG_SMP)
......
...@@ -94,15 +94,15 @@ static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, ...@@ -94,15 +94,15 @@ static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask,
return nodes; return nodes;
} }
static int irq_build_affinity_masks(const struct irq_affinity *affd, static int __irq_build_affinity_masks(const struct irq_affinity *affd,
int startvec, int numvecs, int startvec, int numvecs, int firstvec,
cpumask_var_t *node_to_cpumask, cpumask_var_t *node_to_cpumask,
const struct cpumask *cpu_mask, const struct cpumask *cpu_mask,
struct cpumask *nmsk, struct cpumask *nmsk,
struct cpumask *masks) struct cpumask *masks)
{ {
int n, nodes, cpus_per_vec, extra_vecs, done = 0; int n, nodes, cpus_per_vec, extra_vecs, done = 0;
int last_affv = affd->pre_vectors + numvecs; int last_affv = firstvec + numvecs;
int curvec = startvec; int curvec = startvec;
nodemask_t nodemsk = NODE_MASK_NONE; nodemask_t nodemsk = NODE_MASK_NONE;
...@@ -117,12 +117,11 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, ...@@ -117,12 +117,11 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd,
*/ */
if (numvecs <= nodes) { if (numvecs <= nodes) {
for_each_node_mask(n, nodemsk) { for_each_node_mask(n, nodemsk) {
cpumask_copy(masks + curvec, node_to_cpumask[n]); cpumask_or(masks + curvec, masks + curvec, node_to_cpumask[n]);
if (++done == numvecs)
break;
if (++curvec == last_affv) if (++curvec == last_affv)
curvec = affd->pre_vectors; curvec = firstvec;
} }
done = numvecs;
goto out; goto out;
} }
...@@ -130,7 +129,7 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, ...@@ -130,7 +129,7 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd,
int ncpus, v, vecs_to_assign, vecs_per_node; int ncpus, v, vecs_to_assign, vecs_per_node;
/* Spread the vectors per node */ /* Spread the vectors per node */
vecs_per_node = (numvecs - (curvec - affd->pre_vectors)) / nodes; vecs_per_node = (numvecs - (curvec - firstvec)) / nodes;
/* Get the cpus on this node which are in the mask */ /* Get the cpus on this node which are in the mask */
cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]); cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
...@@ -158,7 +157,7 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, ...@@ -158,7 +157,7 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd,
if (done >= numvecs) if (done >= numvecs)
break; break;
if (curvec >= last_affv) if (curvec >= last_affv)
curvec = affd->pre_vectors; curvec = firstvec;
--nodes; --nodes;
} }
...@@ -166,6 +165,62 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, ...@@ -166,6 +165,62 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd,
return done; return done;
} }
/*
* build affinity in two stages:
* 1) spread present CPU on these vectors
* 2) spread other possible CPUs on these vectors
*/
static int irq_build_affinity_masks(const struct irq_affinity *affd,
int startvec, int numvecs, int firstvec,
cpumask_var_t *node_to_cpumask,
struct cpumask *masks)
{
int curvec = startvec, nr_present, nr_others;
int ret = -ENOMEM;
cpumask_var_t nmsk, npresmsk;
if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
return ret;
if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL))
goto fail;
ret = 0;
/* Stabilize the cpumasks */
get_online_cpus();
build_node_to_cpumask(node_to_cpumask);
/* Spread on present CPUs starting from affd->pre_vectors */
nr_present = __irq_build_affinity_masks(affd, curvec, numvecs,
firstvec, node_to_cpumask,
cpu_present_mask, nmsk, masks);
/*
* Spread on non present CPUs starting from the next vector to be
* handled. If the spreading of present CPUs already exhausted the
* vector space, assign the non present CPUs to the already spread
* out vectors.
*/
if (nr_present >= numvecs)
curvec = firstvec;
else
curvec = firstvec + nr_present;
cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask);
nr_others = __irq_build_affinity_masks(affd, curvec, numvecs,
firstvec, node_to_cpumask,
npresmsk, nmsk, masks);
put_online_cpus();
if (nr_present < numvecs)
WARN_ON(nr_present + nr_others < numvecs);
free_cpumask_var(npresmsk);
fail:
free_cpumask_var(nmsk);
return ret;
}
/** /**
* irq_create_affinity_masks - Create affinity masks for multiqueue spreading * irq_create_affinity_masks - Create affinity masks for multiqueue spreading
* @nvecs: The total number of vectors * @nvecs: The total number of vectors
...@@ -178,8 +233,9 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) ...@@ -178,8 +233,9 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
{ {
int affvecs = nvecs - affd->pre_vectors - affd->post_vectors; int affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
int curvec, usedvecs; int curvec, usedvecs;
cpumask_var_t nmsk, npresmsk, *node_to_cpumask; cpumask_var_t *node_to_cpumask;
struct cpumask *masks = NULL; struct cpumask *masks = NULL;
int i, nr_sets;
/* /*
* If there aren't any vectors left after applying the pre/post * If there aren't any vectors left after applying the pre/post
...@@ -188,15 +244,9 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) ...@@ -188,15 +244,9 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
if (nvecs == affd->pre_vectors + affd->post_vectors) if (nvecs == affd->pre_vectors + affd->post_vectors)
return NULL; return NULL;
if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
return NULL;
if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL))
goto outcpumsk;
node_to_cpumask = alloc_node_to_cpumask(); node_to_cpumask = alloc_node_to_cpumask();
if (!node_to_cpumask) if (!node_to_cpumask)
goto outnpresmsk; return NULL;
masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
if (!masks) if (!masks)
...@@ -206,30 +256,28 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) ...@@ -206,30 +256,28 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
for (curvec = 0; curvec < affd->pre_vectors; curvec++) for (curvec = 0; curvec < affd->pre_vectors; curvec++)
cpumask_copy(masks + curvec, irq_default_affinity); cpumask_copy(masks + curvec, irq_default_affinity);
/* Stabilize the cpumasks */
get_online_cpus();
build_node_to_cpumask(node_to_cpumask);
/* Spread on present CPUs starting from affd->pre_vectors */
usedvecs = irq_build_affinity_masks(affd, curvec, affvecs,
node_to_cpumask, cpu_present_mask,
nmsk, masks);
/* /*
* Spread on non present CPUs starting from the next vector to be * Spread on present CPUs starting from affd->pre_vectors. If we
* handled. If the spreading of present CPUs already exhausted the * have multiple sets, build each sets affinity mask separately.
* vector space, assign the non present CPUs to the already spread
* out vectors.
*/ */
if (usedvecs >= affvecs) nr_sets = affd->nr_sets;
curvec = affd->pre_vectors; if (!nr_sets)
else nr_sets = 1;
curvec = affd->pre_vectors + usedvecs;
cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask); for (i = 0, usedvecs = 0; i < nr_sets; i++) {
usedvecs += irq_build_affinity_masks(affd, curvec, affvecs, int this_vecs = affd->sets ? affd->sets[i] : affvecs;
node_to_cpumask, npresmsk, int ret;
nmsk, masks);
put_online_cpus(); ret = irq_build_affinity_masks(affd, curvec, this_vecs,
curvec, node_to_cpumask, masks);
if (ret) {
kfree(masks);
masks = NULL;
goto outnodemsk;
}
curvec += this_vecs;
usedvecs += this_vecs;
}
/* Fill out vectors at the end that don't need affinity */ /* Fill out vectors at the end that don't need affinity */
if (usedvecs >= affvecs) if (usedvecs >= affvecs)
...@@ -241,10 +289,6 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) ...@@ -241,10 +289,6 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
outnodemsk: outnodemsk:
free_node_to_cpumask(node_to_cpumask); free_node_to_cpumask(node_to_cpumask);
outnpresmsk:
free_cpumask_var(npresmsk);
outcpumsk:
free_cpumask_var(nmsk);
return masks; return masks;
} }
...@@ -258,13 +302,21 @@ int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity ...@@ -258,13 +302,21 @@ int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity
{ {
int resv = affd->pre_vectors + affd->post_vectors; int resv = affd->pre_vectors + affd->post_vectors;
int vecs = maxvec - resv; int vecs = maxvec - resv;
int ret; int set_vecs;
if (resv > minvec) if (resv > minvec)
return 0; return 0;
get_online_cpus(); if (affd->nr_sets) {
ret = min_t(int, cpumask_weight(cpu_possible_mask), vecs) + resv; int i;
put_online_cpus();
return ret; for (i = 0, set_vecs = 0; i < affd->nr_sets; i++)
set_vecs += affd->sets[i];
} else {
get_online_cpus();
set_vecs = cpumask_weight(cpu_possible_mask);
put_online_cpus();
}
return resv + min(set_vecs, vecs);
} }
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment