[Skiboot] [PATCH] xive: Add calls to save/restore the queues and VPs HW state
Michael Ellerman
mpe at ellerman.id.au
Mon Feb 25 14:52:41 AEDT 2019
Cédric Le Goater <clg at kaod.org> writes:
> To be able to support migration of guests using the XIVE native
> exploitation mode, (where the queue is effectively owned by the
> guest), KVM needs to be able to save and restore the HW-modified
> fields of the queue, such as the current queue producer pointer and
> generation bit, and to retrieve the modified thread context registers
> of the VP from the NVT structure : the VP interrupt pending bits.
>
> There is no need to set back the NVT structure on P9. Nevertheless, an
> OPAL call number is provisioned in case this is necessary on P10.
I'm not sure how reserving a call number helps us?
If we want to be forward compatible (I think we do?) then we should be
defining an actual call and the kernel should be calling it today. It
can be implemented as a nop on P9, but that way if it's needed on P10
the kernel will already be calling it and skiboot can just implement it.
cheers
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 5f397c8e6731..86eb3e0c331a 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -199,8 +199,8 @@
> #define OPAL_XIVE_FREE_IRQ 140
> #define OPAL_XIVE_SYNC 141
> #define OPAL_XIVE_DUMP 142
> -#define OPAL_XIVE_RESERVED3 143
> -#define OPAL_XIVE_RESERVED4 144
> +#define OPAL_XIVE_GET_QUEUE_STATE 143 /* Get END state */
> +#define OPAL_XIVE_SET_QUEUE_STATE 144 /* Set END state */
> #define OPAL_SIGNAL_SYSTEM_RESET 145
> #define OPAL_NPU_INIT_CONTEXT 146
> #define OPAL_NPU_DESTROY_CONTEXT 147
> @@ -226,7 +226,9 @@
> #define OPAL_NX_COPROC_INIT 167
> #define OPAL_NPU_SET_RELAXED_ORDER 168
> #define OPAL_NPU_GET_RELAXED_ORDER 169
> -#define OPAL_LAST 169
> +#define OPAL_XIVE_GET_VP_STATE 170 /* Get NVT state */
> +#define OPAL_XIVE_RESERVED 171 /* For Set NVT state */
> +#define OPAL_LAST 171
>
> #define QUIESCE_HOLD 1 /* Spin all calls at entry */
> #define QUIESCE_REJECT 2 /* Fail all calls with OPAL_BUSY */
> diff --git a/hw/xive.c b/hw/xive.c
> index 515f154d7894..c100c3bc84c7 100644
> --- a/hw/xive.c
> +++ b/hw/xive.c
> @@ -1398,11 +1398,21 @@ static int64_t xive_ivc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> return __xive_cache_scrub(x, xive_cache_ivc, block, idx, false, false);
> }
>
> +static int64_t xive_vpc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> +{
> + return __xive_cache_scrub(x, xive_cache_vpc, block, idx, false, false);
> +}
> +
> static int64_t xive_vpc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
> {
> return __xive_cache_scrub(x, xive_cache_vpc, block, idx, true, false);
> }
>
> +static int64_t xive_eqc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> +{
> + return __xive_cache_scrub(x, xive_cache_eqc, block, idx, false, false);
> +}
> +
> static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
> uint64_t block, uint64_t idx,
> uint32_t start_dword, uint32_t dword_count,
> @@ -4191,6 +4201,85 @@ static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
> return rc;
> }
>
> +static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t *out_qtoggle,
> + uint32_t *out_qindex)
> +{
> + uint32_t blk, idx;
> + struct xive *x;
> + struct xive_eq *eq;
> + int64_t rc;
> +
> + if (xive_mode != XIVE_MODE_EXPL)
> + return OPAL_WRONG_STATE;
> +
> + if (!out_qtoggle || !out_qindex ||
> + !xive_eq_for_target(vp, prio, &blk, &idx))
> + return OPAL_PARAMETER;
> +
> + x = xive_from_vc_blk(blk);
> + if (!x)
> + return OPAL_PARAMETER;
> +
> + eq = xive_get_eq(x, idx);
> + if (!eq)
> + return OPAL_PARAMETER;
> +
> + /* Scrub the queue */
> + lock(&x->lock);
> + rc = xive_eqc_scrub(x, blk, idx);
> + unlock(&x->lock);
> + if (rc)
> + return rc;
> +
> + /* We don't do disable queues */
> + if (!(eq->w0 & EQ_W0_VALID))
> + return OPAL_WRONG_STATE;
> +
> + *out_qtoggle = GETFIELD(EQ_W1_GENERATION, eq->w1);
> + *out_qindex = GETFIELD(EQ_W1_PAGE_OFF, eq->w1);
> +
> + return OPAL_SUCCESS;
> +}
> +
> +static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t qtoggle, uint32_t qindex)
> +{
> + uint32_t blk, idx;
> + struct xive *x;
> + struct xive_eq *eq, new_eq;
> + int64_t rc;
> +
> + if (xive_mode != XIVE_MODE_EXPL)
> + return OPAL_WRONG_STATE;
> +
> + if (!xive_eq_for_target(vp, prio, &blk, &idx))
> + return OPAL_PARAMETER;
> +
> + x = xive_from_vc_blk(blk);
> + if (!x)
> + return OPAL_PARAMETER;
> +
> + eq = xive_get_eq(x, idx);
> + if (!eq)
> + return OPAL_PARAMETER;
> +
> + /* We don't do disable queues */
> + if (!(eq->w0 & EQ_W0_VALID))
> + return OPAL_WRONG_STATE;
> +
> + new_eq = *eq;
> +
> + new_eq.w1 = SETFIELD(EQ_W1_GENERATION, new_eq.w1, qtoggle);
> + new_eq.w1 = SETFIELD(EQ_W1_PAGE_OFF, new_eq.w1, qindex);
> +
> + lock(&x->lock);
> + rc = xive_eqc_cache_update(x, blk, idx, 0, 4, &new_eq, false, false);
> + unlock(&x->lock);
> +
> + return rc;
> +}
> +
> static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
> {
> struct proc_chip *c = get_chip(chip_id);
> @@ -4436,6 +4525,48 @@ bail:
> return rc;
> }
>
> +static int64_t opal_xive_get_vp_state(uint64_t vp_id, uint64_t *out_state)
> +{
> + struct xive *x;
> + struct xive_vp *vp;
> + uint32_t blk, idx;
> + int64_t rc;
> + bool group;
> +
> + if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
> + return OPAL_PARAMETER;
> + if (group)
> + return OPAL_PARAMETER;
> + x = xive_from_pc_blk(blk);
> + if (!x)
> + return OPAL_PARAMETER;
> + vp = xive_get_vp(x, idx);
> + if (!vp)
> + return OPAL_PARAMETER;
> +
> + /* Scrub the vp */
> + lock(&x->lock);
> + rc = xive_vpc_scrub(x, blk, idx);
> + unlock(&x->lock);
> + if (rc)
> + return rc;
> +
> + if (!(vp->w0 & VP_W0_VALID))
> + return OPAL_WRONG_STATE;
> +
> + /*
> + * Return word4 and word5 which contain the saved HW thread
> + * context. The IPB register is all we care for now on P9.
> + */
> + *out_state = (((uint64_t)vp->w4) << 32) | vp->w5;
> +
> + return OPAL_SUCCESS;
> +}
> +
> +/*
> + * TODO: P9 does not need to set the VP state but P10 might need to.
> + */
> +
> static void xive_cleanup_cpu_tima(struct cpu_thread *c)
> {
> struct xive_cpu_state *xs = c->xstate;
> @@ -5357,5 +5488,8 @@ void init_xive(void)
> opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
> opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
> opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
> + opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
> + opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
> + opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
> }
>
> diff --git a/doc/xive.rst b/doc/xive.rst
> index 0997c7224e11..cf5444336378 100644
> --- a/doc/xive.rst
> +++ b/doc/xive.rst
> @@ -790,3 +790,53 @@ state information about the XIVE.
> - XIVE_DUMP_EMU: Dump the state of the XICS emulation for a thread
> "id" is the PIR value of the thread
>
> +
> +OPAL_XIVE_GET_QUEUE_STATE
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t *out_qtoggle,
> + uint32_t *out_qindex);
> +
> +This call saves the queue toggle bit and index. This must be called on
> +an enabled queue.
> +
> +* vp, prio: The target queue
> +
> +* out_qtoggle: toggle bit of the queue
> +
> +* out_qindex: index of the queue
> +
> +
> +OPAL_XIVE_SET_QUEUE_STATE
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t qtoggle,
> + uint32_t qindex);
> +
> +This call restores the queue toggle bit and index that was previously
> +saved by a call to opal_xive_get_queue_state(). This must be called on
> +an enabled queue.
> +
> +* vp, prio: The target queue
> +
> +* qtoggle: toggle bit of the queue
> +
> +* qindex: index of the queue
> +
> +OPAL_XIVE_GET_VP_STATE
> +^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_get_vp_state(uint64_t vp_id,
> + uint64_t *out_state);
> +
> +This call saves the VP HW state in "out_state". The format matches the
> +XIVE NVT word 4 and word 5. This must be called on an enabled VP.
> +
> +* vp_id: The target VP
> +
> +* out_state: Location where the state is to be stored
> --
> 2.20.1
>
> _______________________________________________
> Skiboot mailing list
> Skiboot at lists.ozlabs.org
> https://lists.ozlabs.org/listinfo/skiboot
More information about the Skiboot
mailing list