[Skiboot] [PATCH v2] xive: Add calls to save/restore the queues and VPs HW state
Cédric Le Goater
clg at kaod.org
Wed Mar 13 07:48:52 AEDT 2019
Cc: David who has been following closely the KVM XIVE development.
Thanks,
C.
On 2/25/19 11:09 AM, Cédric Le Goater wrote:
> To be able to support migration of guests using the XIVE native
> exploitation mode, (where the queue is effectively owned by the
> guest), KVM needs to be able to save and restore the HW-modified
> fields of the queue, such as the current queue producer pointer and
> generation bit, and to retrieve the modified thread context registers
> of the VP from the NVT structure : the VP interrupt pending bits.
>
> However, there is no need to set back the NVT structure on P9. P10
> should be the same.
>
> Based on previous work from BenH.
>
> Signed-off-by: Cédric Le Goater <clg at kaod.org>
> ---
> include/opal-api.h | 7 +--
> hw/xive.c | 130 +++++++++++++++++++++++++++++++++++++++++++++
> doc/xive.rst | 51 ++++++++++++++++++
> 3 files changed, 185 insertions(+), 3 deletions(-)
>
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 5f397c8e6731..52a61a22be75 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -199,8 +199,8 @@
> #define OPAL_XIVE_FREE_IRQ 140
> #define OPAL_XIVE_SYNC 141
> #define OPAL_XIVE_DUMP 142
> -#define OPAL_XIVE_RESERVED3 143
> -#define OPAL_XIVE_RESERVED4 144
> +#define OPAL_XIVE_GET_QUEUE_STATE 143 /* Get END state */
> +#define OPAL_XIVE_SET_QUEUE_STATE 144 /* Set END state */
> #define OPAL_SIGNAL_SYSTEM_RESET 145
> #define OPAL_NPU_INIT_CONTEXT 146
> #define OPAL_NPU_DESTROY_CONTEXT 147
> @@ -226,7 +226,8 @@
> #define OPAL_NX_COPROC_INIT 167
> #define OPAL_NPU_SET_RELAXED_ORDER 168
> #define OPAL_NPU_GET_RELAXED_ORDER 169
> -#define OPAL_LAST 169
> +#define OPAL_XIVE_GET_VP_STATE 170 /* Get NVT state */
> +#define OPAL_LAST 170
>
> #define QUIESCE_HOLD 1 /* Spin all calls at entry */
> #define QUIESCE_REJECT 2 /* Fail all calls with OPAL_BUSY */
> diff --git a/hw/xive.c b/hw/xive.c
> index 05e5531cbfca..18eb6cc3188d 100644
> --- a/hw/xive.c
> +++ b/hw/xive.c
> @@ -1397,11 +1397,21 @@ static int64_t xive_ivc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> return __xive_cache_scrub(x, xive_cache_ivc, block, idx, false, false);
> }
>
> +static int64_t xive_vpc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> +{
> + return __xive_cache_scrub(x, xive_cache_vpc, block, idx, false, false);
> +}
> +
> static int64_t xive_vpc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
> {
> return __xive_cache_scrub(x, xive_cache_vpc, block, idx, true, false);
> }
>
> +static int64_t xive_eqc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> +{
> + return __xive_cache_scrub(x, xive_cache_eqc, block, idx, false, false);
> +}
> +
> static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
> uint64_t block, uint64_t idx,
> uint32_t start_dword, uint32_t dword_count,
> @@ -4177,6 +4187,85 @@ static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
> return rc;
> }
>
> +static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t *out_qtoggle,
> + uint32_t *out_qindex)
> +{
> + uint32_t blk, idx;
> + struct xive *x;
> + struct xive_eq *eq;
> + int64_t rc;
> +
> + if (xive_mode != XIVE_MODE_EXPL)
> + return OPAL_WRONG_STATE;
> +
> + if (!out_qtoggle || !out_qindex ||
> + !xive_eq_for_target(vp, prio, &blk, &idx))
> + return OPAL_PARAMETER;
> +
> + x = xive_from_vc_blk(blk);
> + if (!x)
> + return OPAL_PARAMETER;
> +
> + eq = xive_get_eq(x, idx);
> + if (!eq)
> + return OPAL_PARAMETER;
> +
> + /* Scrub the queue */
> + lock(&x->lock);
> + rc = xive_eqc_scrub(x, blk, idx);
> + unlock(&x->lock);
> + if (rc)
> + return rc;
> +
> + /* We don't do disable queues */
> + if (!(eq->w0 & EQ_W0_VALID))
> + return OPAL_WRONG_STATE;
> +
> + *out_qtoggle = GETFIELD(EQ_W1_GENERATION, eq->w1);
> + *out_qindex = GETFIELD(EQ_W1_PAGE_OFF, eq->w1);
> +
> + return OPAL_SUCCESS;
> +}
> +
> +static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t qtoggle, uint32_t qindex)
> +{
> + uint32_t blk, idx;
> + struct xive *x;
> + struct xive_eq *eq, new_eq;
> + int64_t rc;
> +
> + if (xive_mode != XIVE_MODE_EXPL)
> + return OPAL_WRONG_STATE;
> +
> + if (!xive_eq_for_target(vp, prio, &blk, &idx))
> + return OPAL_PARAMETER;
> +
> + x = xive_from_vc_blk(blk);
> + if (!x)
> + return OPAL_PARAMETER;
> +
> + eq = xive_get_eq(x, idx);
> + if (!eq)
> + return OPAL_PARAMETER;
> +
> + /* We don't do disable queues */
> + if (!(eq->w0 & EQ_W0_VALID))
> + return OPAL_WRONG_STATE;
> +
> + new_eq = *eq;
> +
> + new_eq.w1 = SETFIELD(EQ_W1_GENERATION, new_eq.w1, qtoggle);
> + new_eq.w1 = SETFIELD(EQ_W1_PAGE_OFF, new_eq.w1, qindex);
> +
> + lock(&x->lock);
> + rc = xive_eqc_cache_update(x, blk, idx, 0, 4, &new_eq, false, false);
> + unlock(&x->lock);
> +
> + return rc;
> +}
> +
> static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
> {
> struct proc_chip *c = get_chip(chip_id);
> @@ -4415,6 +4504,44 @@ bail:
> return rc;
> }
>
> +static int64_t opal_xive_get_vp_state(uint64_t vp_id, uint64_t *out_state)
> +{
> + struct xive *x;
> + struct xive_vp *vp;
> + uint32_t blk, idx;
> + int64_t rc;
> + bool group;
> +
> + if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
> + return OPAL_PARAMETER;
> + if (group)
> + return OPAL_PARAMETER;
> + x = xive_from_pc_blk(blk);
> + if (!x)
> + return OPAL_PARAMETER;
> + vp = xive_get_vp(x, idx);
> + if (!vp)
> + return OPAL_PARAMETER;
> +
> + /* Scrub the vp */
> + lock(&x->lock);
> + rc = xive_vpc_scrub(x, blk, idx);
> + unlock(&x->lock);
> + if (rc)
> + return rc;
> +
> + if (!(vp->w0 & VP_W0_VALID))
> + return OPAL_WRONG_STATE;
> +
> + /*
> + * Return word4 and word5 which contain the saved HW thread
> + * context. The IPB register is all we care for now on P9.
> + */
> + *out_state = (((uint64_t)vp->w4) << 32) | vp->w5;
> +
> + return OPAL_SUCCESS;
> +}
> +
> static void xive_cleanup_cpu_tima(struct cpu_thread *c)
> {
> struct xive_cpu_state *xs = c->xstate;
> @@ -5336,5 +5463,8 @@ void init_xive(void)
> opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
> opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
> opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
> + opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
> + opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
> + opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
> }
>
> diff --git a/doc/xive.rst b/doc/xive.rst
> index 0997c7224e11..42a60907f393 100644
> --- a/doc/xive.rst
> +++ b/doc/xive.rst
> @@ -790,3 +790,54 @@ state information about the XIVE.
> - XIVE_DUMP_EMU: Dump the state of the XICS emulation for a thread
> "id" is the PIR value of the thread
>
> +
> +OPAL_XIVE_GET_QUEUE_STATE
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t *out_qtoggle,
> + uint32_t *out_qindex);
> +
> +This call saves the queue toggle bit and index. This must be called on
> +an enabled queue.
> +
> +* vp, prio: The target queue
> +
> +* out_qtoggle: toggle bit of the queue
> +
> +* out_qindex: index of the queue
> +
> +
> +OPAL_XIVE_SET_QUEUE_STATE
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
> + uint32_t qtoggle,
> + uint32_t qindex);
> +
> +This call restores the queue toggle bit and index that was previously
> +saved by a call to opal_xive_get_queue_state(). This must be called on
> +an enabled queue.
> +
> +* vp, prio: The target queue
> +
> +* qtoggle: toggle bit of the queue
> +
> +* qindex: index of the queue
> +
> +
> +OPAL_XIVE_GET_VP_STATE
> +^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_get_vp_state(uint64_t vp_id,
> + uint64_t *out_state);
> +
> +This call saves the VP HW state in "out_state". The format matches the
> +XIVE NVT word 4 and word 5. This must be called on an enabled VP.
> +
> +* vp_id: The target VP
> +
> +* out_state: Location where the state is to be stored
>
More information about the Skiboot
mailing list