[PATCH 2/4] tools/perf: Fix perf bench epoll to correct usage of affinity for machines with #CPUs > 1K

Ian Rogers irogers at google.com
Wed Apr 6 03:56:34 AEST 2022


On Fri, Apr 1, 2022 at 12:00 PM Athira Rajeev
<atrajeev at linux.vnet.ibm.com> wrote:
>
> perf bench epoll testcase fails on systems with CPU's
> more than 1K.
>
> Testcase: perf bench epoll all
> Result snippet:
> <<>>
> Run summary [PID 106497]: 1399 threads monitoring on 64 file-descriptors for 8 secs.
>
> perf: pthread_create: No such file or directory
> <<>>
>
> In epoll benchmarks (ctl, wait) pthread_create is invoked in do_threads
> from respective bench_epoll_*  function. Though the logs shows direct
> failure from pthread_create, the actual failure is from  "sched_setaffinity"
> returning EINVAL (invalid argument). This happens because the default
> mask size in glibc is 1024. To overcome this 1024 CPUs mask size
> limitation of cpu_set_t, change the mask size using the CPU_*_S macros.
>
> Patch addresses this by fixing all the epoll benchmarks to use
> CPU_ALLOC to allocate cpumask, CPU_ALLOC_SIZE for size, and
> CPU_SET_S to set the mask.
>
> Reported-by: Disha Goel <disgoel at linux.vnet.ibm.com>
> Signed-off-by: Athira Rajeev <atrajeev at linux.vnet.ibm.com>
> ---
>  tools/perf/bench/epoll-ctl.c  | 25 +++++++++++++++++++------
>  tools/perf/bench/epoll-wait.c | 25 +++++++++++++++++++------
>  2 files changed, 38 insertions(+), 12 deletions(-)
>
> diff --git a/tools/perf/bench/epoll-ctl.c b/tools/perf/bench/epoll-ctl.c
> index 1a17ec83d3c4..91c53f6c6d87 100644
> --- a/tools/perf/bench/epoll-ctl.c
> +++ b/tools/perf/bench/epoll-ctl.c
> @@ -222,13 +222,20 @@ static void init_fdmaps(struct worker *w, int pct)
>  static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)
>  {
>         pthread_attr_t thread_attr, *attrp = NULL;
> -       cpu_set_t cpuset;
> +       cpu_set_t *cpuset;
>         unsigned int i, j;
>         int ret = 0;
> +       int nrcpus;
> +       size_t size;
>
>         if (!noaffinity)
>                 pthread_attr_init(&thread_attr);
>
> +       nrcpus = perf_cpu_map__nr(cpu);
> +       cpuset = CPU_ALLOC(nrcpus);
> +       BUG_ON(!cpuset);
> +       size = CPU_ALLOC_SIZE(nrcpus);
> +
>         for (i = 0; i < nthreads; i++) {
>                 struct worker *w = &worker[i];
>
> @@ -252,22 +259,28 @@ static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)
>                         init_fdmaps(w, 50);
>
>                 if (!noaffinity) {
> -                       CPU_ZERO(&cpuset);
> -                       CPU_SET(perf_cpu_map__cpu(cpu, i % perf_cpu_map__nr(cpu)).cpu, &cpuset);
> +                       CPU_ZERO_S(size, cpuset);
> +                       CPU_SET_S(perf_cpu_map__cpu(cpu, i % perf_cpu_map__nr(cpu)).cpu,
> +                                       size, cpuset);
>
> -                       ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(cpu_set_t), &cpuset);
> -                       if (ret)
> +                       ret = pthread_attr_setaffinity_np(&thread_attr, size, cpuset);
> +                       if (ret) {
> +                               CPU_FREE(cpuset);
>                                 err(EXIT_FAILURE, "pthread_attr_setaffinity_np");
> +                       }
>
>                         attrp = &thread_attr;
>                 }
>
>                 ret = pthread_create(&w->thread, attrp, workerfn,
>                                      (void *)(struct worker *) w);
> -               if (ret)
> +               if (ret) {
> +                       CPU_FREE(cpuset);
>                         err(EXIT_FAILURE, "pthread_create");
> +               }
>         }
>
> +       CPU_FREE(cpuset);

A nit here you could CPU_FREE right after the ret = pthread_create...
to make it a bit shorter.

Thanks,
Ian

>         if (!noaffinity)
>                 pthread_attr_destroy(&thread_attr);
>
> diff --git a/tools/perf/bench/epoll-wait.c b/tools/perf/bench/epoll-wait.c
> index 0d1dd8879197..9469a53ffab9 100644
> --- a/tools/perf/bench/epoll-wait.c
> +++ b/tools/perf/bench/epoll-wait.c
> @@ -291,9 +291,11 @@ static void print_summary(void)
>  static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)
>  {
>         pthread_attr_t thread_attr, *attrp = NULL;
> -       cpu_set_t cpuset;
> +       cpu_set_t *cpuset;
>         unsigned int i, j;
>         int ret = 0, events = EPOLLIN;
> +       int nrcpus;
> +       size_t size;
>
>         if (oneshot)
>                 events |= EPOLLONESHOT;
> @@ -306,6 +308,11 @@ static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)
>         if (!noaffinity)
>                 pthread_attr_init(&thread_attr);
>
> +       nrcpus = perf_cpu_map__nr(cpu);
> +       cpuset = CPU_ALLOC(nrcpus);
> +       BUG_ON(!cpuset);
> +       size = CPU_ALLOC_SIZE(nrcpus);
> +
>         for (i = 0; i < nthreads; i++) {
>                 struct worker *w = &worker[i];
>
> @@ -341,22 +348,28 @@ static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)
>                 }
>
>                 if (!noaffinity) {
> -                       CPU_ZERO(&cpuset);
> -                       CPU_SET(perf_cpu_map__cpu(cpu, i % perf_cpu_map__nr(cpu)).cpu, &cpuset);
> +                       CPU_ZERO_S(size, cpuset);
> +                       CPU_SET_S(perf_cpu_map__cpu(cpu, i % perf_cpu_map__nr(cpu)).cpu,
> +                                       size, cpuset);
>
> -                       ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(cpu_set_t), &cpuset);
> -                       if (ret)
> +                       ret = pthread_attr_setaffinity_np(&thread_attr, size, cpuset);
> +                       if (ret) {
> +                               CPU_FREE(cpuset);
>                                 err(EXIT_FAILURE, "pthread_attr_setaffinity_np");
> +                       }
>
>                         attrp = &thread_attr;
>                 }
>
>                 ret = pthread_create(&w->thread, attrp, workerfn,
>                                      (void *)(struct worker *) w);
> -               if (ret)
> +               if (ret) {
> +                       CPU_FREE(cpuset);
>                         err(EXIT_FAILURE, "pthread_create");
> +               }
>         }
>
> +       CPU_FREE(cpuset);
>         if (!noaffinity)
>                 pthread_attr_destroy(&thread_attr);
>
> --
> 2.35.1
>


More information about the Linuxppc-dev mailing list