[Skiboot] [PATCH] xive: Add calls to save/restore the queues and VPs HW state

Cédric Le Goater clg at kaod.org
Mon Feb 25 20:27:43 AEDT 2019


On 2/25/19 4:52 AM, Michael Ellerman wrote:
> Cédric Le Goater <clg at kaod.org> writes:
>> To be able to support migration of guests using the XIVE native
>> exploitation mode, (where the queue is effectively owned by the
>> guest), KVM needs to be able to save and restore the HW-modified
>> fields of the queue, such as the current queue producer pointer and
>> generation bit, and to retrieve the modified thread context registers
>> of the VP from the NVT structure : the VP interrupt pending bits.
>>
>> There is no need to set back the NVT structure on P9. Nevertheless, an
>> OPAL call number is provisioned in case this is necessary on P10.
> 
> I'm not sure how reserving a call number helps us?

It is not required that much. This is just a question of aesthetic and 
symmetry.

> If we want to be forward compatible (I think we do?) then we should be
> defining an actual call and the kernel should be calling it today. It
> can be implemented as a nop on P9, but that way if it's needed on P10
> the kernel will already be calling it and skiboot can just implement it.

Yes. I will send a v2 without the reservation for the OPAL calls. It's 
gating the Linux patches. We will need to get these changes in first.

Thanks,

C.

> cheers
> 
>> diff --git a/include/opal-api.h b/include/opal-api.h
>> index 5f397c8e6731..86eb3e0c331a 100644
>> --- a/include/opal-api.h
>> +++ b/include/opal-api.h
>> @@ -199,8 +199,8 @@
>>  #define OPAL_XIVE_FREE_IRQ			140
>>  #define OPAL_XIVE_SYNC				141
>>  #define OPAL_XIVE_DUMP				142
>> -#define OPAL_XIVE_RESERVED3			143
>> -#define OPAL_XIVE_RESERVED4			144
>> +#define OPAL_XIVE_GET_QUEUE_STATE		143 /* Get END state */
>> +#define OPAL_XIVE_SET_QUEUE_STATE		144 /* Set END state */
>>  #define OPAL_SIGNAL_SYSTEM_RESET		145
>>  #define OPAL_NPU_INIT_CONTEXT			146
>>  #define OPAL_NPU_DESTROY_CONTEXT		147
>> @@ -226,7 +226,9 @@
>>  #define OPAL_NX_COPROC_INIT			167
>>  #define OPAL_NPU_SET_RELAXED_ORDER		168
>>  #define OPAL_NPU_GET_RELAXED_ORDER		169
>> -#define OPAL_LAST				169
>> +#define OPAL_XIVE_GET_VP_STATE			170 /* Get NVT state */
>> +#define OPAL_XIVE_RESERVED			171 /* For Set NVT state */
>> +#define OPAL_LAST				171
>>  
>>  #define QUIESCE_HOLD			1 /* Spin all calls at entry */
>>  #define QUIESCE_REJECT			2 /* Fail all calls with OPAL_BUSY */
>> diff --git a/hw/xive.c b/hw/xive.c
>> index 515f154d7894..c100c3bc84c7 100644
>> --- a/hw/xive.c
>> +++ b/hw/xive.c
>> @@ -1398,11 +1398,21 @@ static int64_t xive_ivc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>>  	return __xive_cache_scrub(x, xive_cache_ivc, block, idx, false, false);
>>  }
>>  
>> +static int64_t xive_vpc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>> +{
>> +	return __xive_cache_scrub(x, xive_cache_vpc, block, idx, false, false);
>> +}
>> +
>>  static int64_t xive_vpc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
>>  {
>>  	return __xive_cache_scrub(x, xive_cache_vpc, block, idx, true, false);
>>  }
>>  
>> +static int64_t xive_eqc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>> +{
>> +	return __xive_cache_scrub(x, xive_cache_eqc, block, idx, false, false);
>> +}
>> +
>>  static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
>>  				  uint64_t block, uint64_t idx,
>>  				  uint32_t start_dword, uint32_t dword_count,
>> @@ -4191,6 +4201,85 @@ static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
>>  	return rc;
>>  }
>>  
>> +static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
>> +					 uint32_t *out_qtoggle,
>> +					 uint32_t *out_qindex)
>> +{
>> +	uint32_t blk, idx;
>> +	struct xive *x;
>> +	struct xive_eq *eq;
>> +	int64_t rc;
>> +
>> +	if (xive_mode != XIVE_MODE_EXPL)
>> +		return OPAL_WRONG_STATE;
>> +
>> +	if (!out_qtoggle || !out_qindex ||
>> +	    !xive_eq_for_target(vp, prio, &blk, &idx))
>> +		return OPAL_PARAMETER;
>> +
>> +	x = xive_from_vc_blk(blk);
>> +	if (!x)
>> +		return OPAL_PARAMETER;
>> +
>> +	eq = xive_get_eq(x, idx);
>> +	if (!eq)
>> +		return OPAL_PARAMETER;
>> +
>> +	/* Scrub the queue */
>> +	lock(&x->lock);
>> +	rc = xive_eqc_scrub(x, blk, idx);
>> +	unlock(&x->lock);
>> +	if (rc)
>> +		return rc;
>> +
>> +	/* We don't do disable queues */
>> +	if (!(eq->w0 & EQ_W0_VALID))
>> +		return OPAL_WRONG_STATE;
>> +
>> +	*out_qtoggle = GETFIELD(EQ_W1_GENERATION, eq->w1);
>> +	*out_qindex  = GETFIELD(EQ_W1_PAGE_OFF, eq->w1);
>> +
>> +	return OPAL_SUCCESS;
>> +}
>> +
>> +static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
>> +					 uint32_t qtoggle, uint32_t qindex)
>> +{
>> +	uint32_t blk, idx;
>> +	struct xive *x;
>> +	struct xive_eq *eq, new_eq;
>> +	int64_t rc;
>> +
>> +	if (xive_mode != XIVE_MODE_EXPL)
>> +		return OPAL_WRONG_STATE;
>> +
>> +	if (!xive_eq_for_target(vp, prio, &blk, &idx))
>> +		return OPAL_PARAMETER;
>> +
>> +	x = xive_from_vc_blk(blk);
>> +	if (!x)
>> +		return OPAL_PARAMETER;
>> +
>> +	eq = xive_get_eq(x, idx);
>> +	if (!eq)
>> +		return OPAL_PARAMETER;
>> +
>> +	/* We don't do disable queues */
>> +	if (!(eq->w0 & EQ_W0_VALID))
>> +		return OPAL_WRONG_STATE;
>> +
>> +	new_eq = *eq;
>> +
>> +	new_eq.w1 = SETFIELD(EQ_W1_GENERATION, new_eq.w1, qtoggle);
>> +	new_eq.w1 = SETFIELD(EQ_W1_PAGE_OFF, new_eq.w1, qindex);
>> +
>> +	lock(&x->lock);
>> +	rc = xive_eqc_cache_update(x, blk, idx, 0, 4, &new_eq, false, false);
>> +	unlock(&x->lock);
>> +
>> +	return rc;
>> +}
>> +
>>  static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
>>  {
>>  	struct proc_chip *c = get_chip(chip_id);
>> @@ -4436,6 +4525,48 @@ bail:
>>  	return rc;
>>  }
>>  
>> +static int64_t opal_xive_get_vp_state(uint64_t vp_id, uint64_t *out_state)
>> +{
>> +	struct xive *x;
>> +	struct xive_vp *vp;
>> +	uint32_t blk, idx;
>> +	int64_t rc;
>> +	bool group;
>> +
>> +	if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
>> +		return OPAL_PARAMETER;
>> +	if (group)
>> +		return OPAL_PARAMETER;
>> +	x = xive_from_pc_blk(blk);
>> +	if (!x)
>> +		return OPAL_PARAMETER;
>> +	vp = xive_get_vp(x, idx);
>> +	if (!vp)
>> +		return OPAL_PARAMETER;
>> +
>> +	/* Scrub the vp */
>> +	lock(&x->lock);
>> +	rc = xive_vpc_scrub(x, blk, idx);
>> +	unlock(&x->lock);
>> +	if (rc)
>> +		return rc;
>> +
>> +	if (!(vp->w0 & VP_W0_VALID))
>> +		return OPAL_WRONG_STATE;
>> +
>> +	/*
>> +	 * Return word4 and word5 which contain the saved HW thread
>> +	 * context. The IPB register is all we care for now on P9.
>> +	 */
>> +	*out_state = (((uint64_t)vp->w4) << 32) | vp->w5;
>> +
>> +	return OPAL_SUCCESS;
>> +}
>> +
>> +/*
>> + * TODO: P9 does not need to set the VP state but P10 might need to.
>> + */
>> +
>>  static void xive_cleanup_cpu_tima(struct cpu_thread *c)
>>  {
>>  	struct xive_cpu_state *xs = c->xstate;
>> @@ -5357,5 +5488,8 @@ void init_xive(void)
>>  	opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
>>  	opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
>>  	opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
>> +	opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
>> +	opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
>> +	opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
>>  }
>>  
>> diff --git a/doc/xive.rst b/doc/xive.rst
>> index 0997c7224e11..cf5444336378 100644
>> --- a/doc/xive.rst
>> +++ b/doc/xive.rst
>> @@ -790,3 +790,53 @@ state information about the XIVE.
>>    - XIVE_DUMP_EMU:     Dump the state of the XICS emulation for a thread
>>  		       "id" is the PIR value of the thread
>>  
>> +
>> +OPAL_XIVE_GET_QUEUE_STATE
>> +^^^^^^^^^^^^^^^^^^^^^^^^^
>> +.. code-block:: c
>> +
>> + int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
>> +				   uint32_t *out_qtoggle,
>> +				   uint32_t *out_qindex);
>> +
>> +This call saves the queue toggle bit and index. This must be called on
>> +an enabled queue.
>> +
>> +* vp, prio: The target queue
>> +
>> +* out_qtoggle: toggle bit of the queue
>> +
>> +* out_qindex: index of the queue
>> +
>> +
>> +OPAL_XIVE_SET_QUEUE_STATE
>> +^^^^^^^^^^^^^^^^^^^^^^^^^
>> +.. code-block:: c
>> +
>> + int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
>> +				   uint32_t qtoggle,
>> +				   uint32_t qindex);
>> +
>> +This call restores the queue toggle bit and index that was previously
>> +saved by a call to opal_xive_get_queue_state(). This must be called on
>> +an enabled queue.
>> +
>> +* vp, prio: The target queue
>> +
>> +* qtoggle: toggle bit of the queue
>> +
>> +* qindex: index of the queue
>> +
>> +OPAL_XIVE_GET_VP_STATE
>> +^^^^^^^^^^^^^^^^^^^^^^
>> +.. code-block:: c
>> +
>> + int64_t opal_xive_get_vp_state(uint64_t vp_id,
>> +				uint64_t *out_state);
>> +
>> +This call saves the VP HW state in "out_state". The format matches the
>> +XIVE NVT word 4 and word 5. This must be called on an enabled VP.
>> +
>> +* vp_id: The target VP
>> +
>> +* out_state: Location where the state is to be stored
>> -- 
>> 2.20.1
>>
>> _______________________________________________
>> Skiboot mailing list
>> Skiboot at lists.ozlabs.org
>> https://lists.ozlabs.org/listinfo/skiboot



More information about the Skiboot mailing list