[RFC] cxl: Enable use of custom PE-id for contexts
Andrew Donnellan
andrew.donnellan at au1.ibm.com
Mon Aug 28 15:35:51 AEST 2017
On 28/08/17 14:15, Vaibhav Jain wrote:
> Some AFUs differentiate between multiple PE's based on their
> PE-ids. Unfortunately presently cxl provides no way for external
> users (libcxl/kernel-api) to control the id assigned to a specific
> PE. The ids are currently assigned serially to PE's and are dolled out
> using an idr table.
>
> With this patch external users of the cxl api can request a specific
> __u16 peid to be used when issuing CXL_IOCTL_START_WORK or calling
> cxl_start_context(). Struct cxl_ioctl_start_work has a new member named
> use_pe that can be used to provide the PE-id to be used. Also a new
> CXL_START_WORK flag named CXL_START_WORK_CUSTOM_PE has been introduced
> to indicated to cxl to try to switch the PE-id of the context to the
> value provided in use_pe before attaching the context to the AFU.
>
> The switching of context PE-id is done by inserting the
> same context with the requested PE-id from struct cxl_ioctl_start_work
> and removing the previously allocated PE-id from the afu->contexts_idr
> table. In case an existing context with the requested PE-id already
> exists the first step above will fail causing the context attach to
> abort and an error is returned back to caller.
>
> Signed-off-by: Vaibhav Jain <vaibhav at linux.vnet.ibm.com>
It's a shame we can't do this earlier in cxl_context_init() (because I
don't think there's a way for userspace to pass the pe number in a
simple file open...)
> ---
> drivers/misc/cxl/api.c | 7 ++++++
> drivers/misc/cxl/context.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++
> drivers/misc/cxl/cxl.h | 3 +++
> drivers/misc/cxl/file.c | 12 ++++++++--
> include/uapi/misc/cxl.h | 9 ++++----
> 5 files changed, 80 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
> index 1a138c83f877..3c34f486a0ce 100644
> --- a/drivers/misc/cxl/api.c
> +++ b/drivers/misc/cxl/api.c
> @@ -505,6 +505,13 @@ int cxl_start_work(struct cxl_context *ctx,
> {
> int rc;
>
> + /* If requesting a custom PE id ? */
> + if (work->flags & CXL_START_WORK_CUSTOM_PE) {
> + rc = cxl_context_switch_pe(ctx, work->use_pe);
> + if (rc)
> + return rc;
> + }
> +
> /* code taken from afu_ioctl_start_work */
> if (!(work->flags & CXL_START_WORK_NUM_IRQS))
> work->num_interrupts = ctx->afu->pp_irqs;
> diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
> index 8c32040b9c09..3699891a84be 100644
> --- a/drivers/misc/cxl/context.c
> +++ b/drivers/misc/cxl/context.c
> @@ -117,6 +117,61 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
> return 0;
> }
>
> +/*
> + * Switch the PE# if context ctx to newpe
> + */
> +int cxl_context_switch_pe(struct cxl_context *ctx, __u16 newpe)
> +{
> + int rc, oldpe = 0;
> +
> + pr_debug("Switching PE#%u to PE#%u\n", ctx->pe, newpe);
> + /* Assigning custom PE only supported on bare-metal right now */
> + if (!cpu_has_feature(CPU_FTR_HVMODE)) {
> + pr_err("Assigning custom PE# only supported on bare metal\n");
> + return -EINVAL;
> + }
> +
> + /* Silently ignore if newpe == ctx->pe */
> + if (newpe == ctx->pe)
> + return 0;
> +
> + /* Bunch of sanity validations */
> + if (newpe < ctx->afu->adapter->min_pe ||
> + newpe >= (ctx->afu->adapter->min_pe + ctx->afu->num_procs)) {
> + pr_debug("Cant assign out of range PE#%u\n", newpe);
Can't
> + return -EINVAL;
> + }
> +
> + if (ctx->pe_inserted) {
> + pr_debug("Cant switch PE for attached PE#%u\n", ctx->pe);
Can't
> + return -EINVAL;
> + }
> +
> + /* serialize access to the idr table */
> + mutex_lock(&ctx->afu->contexts_lock);
> + idr_preload(GFP_KERNEL);
> + rc = idr_alloc(&ctx->afu->contexts_idr, ctx, newpe, newpe + 1,
> + GFP_NOWAIT);
> + /* if successful allocation then remove the existing idr entry */
> + if (rc >= 0) {
> + idr_remove(&ctx->afu->contexts_idr, ctx->pe);
> + oldpe = ctx->pe;
> + ctx->pe = newpe;
> + ctx->external_pe = ctx->pe;
> + ctx->elem = &ctx->afu->native->spa[newpe];
> + }
> + idr_preload_end();
> + mutex_unlock(&ctx->afu->contexts_lock);
> +
> + if (rc >= 0)
> + pr_debug("Switched PE#%u -> PE%u\n", oldpe, ctx->pe);
> + else
> + pr_debug("Unable to switch PE#%u->PE%u. Error=%d\n",
> + oldpe, ctx->pe, rc);
> +
> + return rc >= 0 ? 0 : rc;
> +}
> +
> void cxl_context_set_mapping(struct cxl_context *ctx,
> struct address_space *mapping)
> {
> diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
> index b1afeccbb97f..6d7c0accb30e 100644
> --- a/drivers/misc/cxl/cxl.h
> +++ b/drivers/misc/cxl/cxl.h
> @@ -1171,4 +1171,7 @@ void cxl_context_mm_count_get(struct cxl_context *ctx);
> /* Decrements the reference count to "struct mm_struct" */
> void cxl_context_mm_count_put(struct cxl_context *ctx);
>
> +/* Switch the PE# of given context ctx to newpe */
> +int cxl_context_switch_pe(struct cxl_context *ctx, __u16 newpe);
> +
> #endif
> diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
> index 0761271d68c5..7ae7020aa062 100644
> --- a/drivers/misc/cxl/file.c
> +++ b/drivers/misc/cxl/file.c
> @@ -173,12 +173,20 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
> * flags are set it's invalid
> */
> if (work.reserved1 || work.reserved2 || work.reserved3 ||
> - work.reserved4 || work.reserved5 || work.reserved6 ||
> - (work.flags & ~CXL_START_WORK_ALL)) {
> + work.reserved4 || work.reserved5 ||
> + ((work.flags & ~CXL_START_WORK_ALL) &&
> + !(work.flags & CXL_START_WORK_CUSTOM_PE))) {
|| (work.flags & ~(CXL_START_WORK_ALL | CXL_START_WORK_CUSTOM_PE))
Though maybe we should add it to _ALL?
> rc = -EINVAL;
> goto out;
> }
>
> + /* If requesting a custom PE id ? */
> + if (work.flags & CXL_START_WORK_CUSTOM_PE) {
> + rc = cxl_context_switch_pe(ctx, work.use_pe);
> + if (rc)
> + goto out;
> + }
> +
> if (!(work.flags & CXL_START_WORK_NUM_IRQS))
> work.num_interrupts = ctx->afu->pp_irqs;
> else if ((work.num_interrupts < ctx->afu->pp_irqs) ||
> diff --git a/include/uapi/misc/cxl.h b/include/uapi/misc/cxl.h
> index 180d526a55c3..0d585dc46369 100644
> --- a/include/uapi/misc/cxl.h
> +++ b/include/uapi/misc/cxl.h
> @@ -19,18 +19,19 @@ struct cxl_ioctl_start_work {
> __u64 work_element_descriptor;
> __u64 amr;
> __s16 num_interrupts;
> - __s16 reserved1;
> - __s32 reserved2;
> + __u16 use_pe;
> + __s32 reserved1;
> + __u64 reserved2;
> __u64 reserved3;
> __u64 reserved4;
> __u64 reserved5;
> - __u64 reserved6;
> };
>
> #define CXL_START_WORK_AMR 0x0000000000000001ULL
> #define CXL_START_WORK_NUM_IRQS 0x0000000000000002ULL
> #define CXL_START_WORK_ERR_FF 0x0000000000000004ULL
> -#define CXL_START_WORK_ALL (CXL_START_WORK_AMR |\
> +#define CXL_START_WORK_CUSTOM_PE 0x0000000000000008ULL
> +#define CXL_START_WORK_ALL (CXL_START_WORK_AMR | \
> CXL_START_WORK_NUM_IRQS |\
> CXL_START_WORK_ERR_FF)
>
>
--
Andrew Donnellan OzLabs, ADL Canberra
andrew.donnellan at au1.ibm.com IBM Australia Limited
More information about the Linuxppc-dev
mailing list