[Skiboot] [PATCH v2] xive: Add calls to save/restore the queues and VPs HW state

Cédric Le Goater clg at kaod.org
Wed Mar 13 07:48:52 AEDT 2019


Cc: David who has been following closely the KVM XIVE development.

Thanks,

C. 

On 2/25/19 11:09 AM, Cédric Le Goater wrote:
> To be able to support migration of guests using the XIVE native
> exploitation mode, (where the queue is effectively owned by the
> guest), KVM needs to be able to save and restore the HW-modified
> fields of the queue, such as the current queue producer pointer and
> generation bit, and to retrieve the modified thread context registers
> of the VP from the NVT structure : the VP interrupt pending bits.
> 
> However, there is no need to set back the NVT structure on P9. P10
> should be the same.
> 
> Based on previous work from BenH.
> 
> Signed-off-by: Cédric Le Goater <clg at kaod.org>
> ---
>  include/opal-api.h |   7 +--
>  hw/xive.c          | 130 +++++++++++++++++++++++++++++++++++++++++++++
>  doc/xive.rst       |  51 ++++++++++++++++++
>  3 files changed, 185 insertions(+), 3 deletions(-)
> 
> diff --git a/include/opal-api.h b/include/opal-api.h
> index 5f397c8e6731..52a61a22be75 100644
> --- a/include/opal-api.h
> +++ b/include/opal-api.h
> @@ -199,8 +199,8 @@
>  #define OPAL_XIVE_FREE_IRQ			140
>  #define OPAL_XIVE_SYNC				141
>  #define OPAL_XIVE_DUMP				142
> -#define OPAL_XIVE_RESERVED3			143
> -#define OPAL_XIVE_RESERVED4			144
> +#define OPAL_XIVE_GET_QUEUE_STATE		143 /* Get END state */
> +#define OPAL_XIVE_SET_QUEUE_STATE		144 /* Set END state */
>  #define OPAL_SIGNAL_SYSTEM_RESET		145
>  #define OPAL_NPU_INIT_CONTEXT			146
>  #define OPAL_NPU_DESTROY_CONTEXT		147
> @@ -226,7 +226,8 @@
>  #define OPAL_NX_COPROC_INIT			167
>  #define OPAL_NPU_SET_RELAXED_ORDER		168
>  #define OPAL_NPU_GET_RELAXED_ORDER		169
> -#define OPAL_LAST				169
> +#define OPAL_XIVE_GET_VP_STATE			170 /* Get NVT state */
> +#define OPAL_LAST				170
>  
>  #define QUIESCE_HOLD			1 /* Spin all calls at entry */
>  #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
> diff --git a/hw/xive.c b/hw/xive.c
> index 05e5531cbfca..18eb6cc3188d 100644
> --- a/hw/xive.c
> +++ b/hw/xive.c
> @@ -1397,11 +1397,21 @@ static int64_t xive_ivc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>  	return __xive_cache_scrub(x, xive_cache_ivc, block, idx, false, false);
>  }
>  
> +static int64_t xive_vpc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> +{
> +	return __xive_cache_scrub(x, xive_cache_vpc, block, idx, false, false);
> +}
> +
>  static int64_t xive_vpc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
>  {
>  	return __xive_cache_scrub(x, xive_cache_vpc, block, idx, true, false);
>  }
>  
> +static int64_t xive_eqc_scrub(struct xive *x, uint64_t block, uint64_t idx)
> +{
> +	return __xive_cache_scrub(x, xive_cache_eqc, block, idx, false, false);
> +}
> +
>  static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
>  				  uint64_t block, uint64_t idx,
>  				  uint32_t start_dword, uint32_t dword_count,
> @@ -4177,6 +4187,85 @@ static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
>  	return rc;
>  }
>  
> +static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
> +					 uint32_t *out_qtoggle,
> +					 uint32_t *out_qindex)
> +{
> +	uint32_t blk, idx;
> +	struct xive *x;
> +	struct xive_eq *eq;
> +	int64_t rc;
> +
> +	if (xive_mode != XIVE_MODE_EXPL)
> +		return OPAL_WRONG_STATE;
> +
> +	if (!out_qtoggle || !out_qindex ||
> +	    !xive_eq_for_target(vp, prio, &blk, &idx))
> +		return OPAL_PARAMETER;
> +
> +	x = xive_from_vc_blk(blk);
> +	if (!x)
> +		return OPAL_PARAMETER;
> +
> +	eq = xive_get_eq(x, idx);
> +	if (!eq)
> +		return OPAL_PARAMETER;
> +
> +	/* Scrub the queue */
> +	lock(&x->lock);
> +	rc = xive_eqc_scrub(x, blk, idx);
> +	unlock(&x->lock);
> +	if (rc)
> +		return rc;
> +
> +	/* We don't do disable queues */
> +	if (!(eq->w0 & EQ_W0_VALID))
> +		return OPAL_WRONG_STATE;
> +
> +	*out_qtoggle = GETFIELD(EQ_W1_GENERATION, eq->w1);
> +	*out_qindex  = GETFIELD(EQ_W1_PAGE_OFF, eq->w1);
> +
> +	return OPAL_SUCCESS;
> +}
> +
> +static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
> +					 uint32_t qtoggle, uint32_t qindex)
> +{
> +	uint32_t blk, idx;
> +	struct xive *x;
> +	struct xive_eq *eq, new_eq;
> +	int64_t rc;
> +
> +	if (xive_mode != XIVE_MODE_EXPL)
> +		return OPAL_WRONG_STATE;
> +
> +	if (!xive_eq_for_target(vp, prio, &blk, &idx))
> +		return OPAL_PARAMETER;
> +
> +	x = xive_from_vc_blk(blk);
> +	if (!x)
> +		return OPAL_PARAMETER;
> +
> +	eq = xive_get_eq(x, idx);
> +	if (!eq)
> +		return OPAL_PARAMETER;
> +
> +	/* We don't do disable queues */
> +	if (!(eq->w0 & EQ_W0_VALID))
> +		return OPAL_WRONG_STATE;
> +
> +	new_eq = *eq;
> +
> +	new_eq.w1 = SETFIELD(EQ_W1_GENERATION, new_eq.w1, qtoggle);
> +	new_eq.w1 = SETFIELD(EQ_W1_PAGE_OFF, new_eq.w1, qindex);
> +
> +	lock(&x->lock);
> +	rc = xive_eqc_cache_update(x, blk, idx, 0, 4, &new_eq, false, false);
> +	unlock(&x->lock);
> +
> +	return rc;
> +}
> +
>  static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
>  {
>  	struct proc_chip *c = get_chip(chip_id);
> @@ -4415,6 +4504,44 @@ bail:
>  	return rc;
>  }
>  
> +static int64_t opal_xive_get_vp_state(uint64_t vp_id, uint64_t *out_state)
> +{
> +	struct xive *x;
> +	struct xive_vp *vp;
> +	uint32_t blk, idx;
> +	int64_t rc;
> +	bool group;
> +
> +	if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
> +		return OPAL_PARAMETER;
> +	if (group)
> +		return OPAL_PARAMETER;
> +	x = xive_from_pc_blk(blk);
> +	if (!x)
> +		return OPAL_PARAMETER;
> +	vp = xive_get_vp(x, idx);
> +	if (!vp)
> +		return OPAL_PARAMETER;
> +
> +	/* Scrub the vp */
> +	lock(&x->lock);
> +	rc = xive_vpc_scrub(x, blk, idx);
> +	unlock(&x->lock);
> +	if (rc)
> +		return rc;
> +
> +	if (!(vp->w0 & VP_W0_VALID))
> +		return OPAL_WRONG_STATE;
> +
> +	/*
> +	 * Return word4 and word5 which contain the saved HW thread
> +	 * context. The IPB register is all we care for now on P9.
> +	 */
> +	*out_state = (((uint64_t)vp->w4) << 32) | vp->w5;
> +
> +	return OPAL_SUCCESS;
> +}
> +
>  static void xive_cleanup_cpu_tima(struct cpu_thread *c)
>  {
>  	struct xive_cpu_state *xs = c->xstate;
> @@ -5336,5 +5463,8 @@ void init_xive(void)
>  	opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
>  	opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
>  	opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
> +	opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
> +	opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
> +	opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
>  }
>  
> diff --git a/doc/xive.rst b/doc/xive.rst
> index 0997c7224e11..42a60907f393 100644
> --- a/doc/xive.rst
> +++ b/doc/xive.rst
> @@ -790,3 +790,54 @@ state information about the XIVE.
>    - XIVE_DUMP_EMU:     Dump the state of the XICS emulation for a thread
>  		       "id" is the PIR value of the thread
>  
> +
> +OPAL_XIVE_GET_QUEUE_STATE
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
> +				   uint32_t *out_qtoggle,
> +				   uint32_t *out_qindex);
> +
> +This call saves the queue toggle bit and index. This must be called on
> +an enabled queue.
> +
> +* vp, prio: The target queue
> +
> +* out_qtoggle: toggle bit of the queue
> +
> +* out_qindex: index of the queue
> +
> +
> +OPAL_XIVE_SET_QUEUE_STATE
> +^^^^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
> +				   uint32_t qtoggle,
> +				   uint32_t qindex);
> +
> +This call restores the queue toggle bit and index that was previously
> +saved by a call to opal_xive_get_queue_state(). This must be called on
> +an enabled queue.
> +
> +* vp, prio: The target queue
> +
> +* qtoggle: toggle bit of the queue
> +
> +* qindex: index of the queue
> +
> +
> +OPAL_XIVE_GET_VP_STATE
> +^^^^^^^^^^^^^^^^^^^^^^
> +.. code-block:: c
> +
> + int64_t opal_xive_get_vp_state(uint64_t vp_id,
> +				uint64_t *out_state);
> +
> +This call saves the VP HW state in "out_state". The format matches the
> +XIVE NVT word 4 and word 5. This must be called on an enabled VP.
> +
> +* vp_id: The target VP
> +
> +* out_state: Location where the state is to be stored
> 



More information about the Skiboot mailing list