[Skiboot] [PATCH] xive: Add calls to save/restore the queues and VPs HW state
Cédric Le Goater
clg at kaod.org
Mon Feb 25 20:27:43 AEDT 2019
On 2/25/19 4:52 AM, Michael Ellerman wrote:
> Cédric Le Goater <clg at kaod.org> writes:
>> To be able to support migration of guests using the XIVE native
>> exploitation mode, (where the queue is effectively owned by the
>> guest), KVM needs to be able to save and restore the HW-modified
>> fields of the queue, such as the current queue producer pointer and
>> generation bit, and to retrieve the modified thread context registers
>> of the VP from the NVT structure : the VP interrupt pending bits.
>>
>> There is no need to set back the NVT structure on P9. Nevertheless, an
>> OPAL call number is provisioned in case this is necessary on P10.
>
> I'm not sure how reserving a call number helps us?
It is not required that much. This is just a question of aesthetic and
symmetry.
> If we want to be forward compatible (I think we do?) then we should be
> defining an actual call and the kernel should be calling it today. It
> can be implemented as a nop on P9, but that way if it's needed on P10
> the kernel will already be calling it and skiboot can just implement it.
Yes. I will send a v2 without the reservation for the OPAL calls. It's
gating the Linux patches. We will need to get these changes in first.
Thanks,
C.
> cheers
>
>> diff --git a/include/opal-api.h b/include/opal-api.h
>> index 5f397c8e6731..86eb3e0c331a 100644
>> --- a/include/opal-api.h
>> +++ b/include/opal-api.h
>> @@ -199,8 +199,8 @@
>> #define OPAL_XIVE_FREE_IRQ 140
>> #define OPAL_XIVE_SYNC 141
>> #define OPAL_XIVE_DUMP 142
>> -#define OPAL_XIVE_RESERVED3 143
>> -#define OPAL_XIVE_RESERVED4 144
>> +#define OPAL_XIVE_GET_QUEUE_STATE 143 /* Get END state */
>> +#define OPAL_XIVE_SET_QUEUE_STATE 144 /* Set END state */
>> #define OPAL_SIGNAL_SYSTEM_RESET 145
>> #define OPAL_NPU_INIT_CONTEXT 146
>> #define OPAL_NPU_DESTROY_CONTEXT 147
>> @@ -226,7 +226,9 @@
>> #define OPAL_NX_COPROC_INIT 167
>> #define OPAL_NPU_SET_RELAXED_ORDER 168
>> #define OPAL_NPU_GET_RELAXED_ORDER 169
>> -#define OPAL_LAST 169
>> +#define OPAL_XIVE_GET_VP_STATE 170 /* Get NVT state */
>> +#define OPAL_XIVE_RESERVED 171 /* For Set NVT state */
>> +#define OPAL_LAST 171
>>
>> #define QUIESCE_HOLD 1 /* Spin all calls at entry */
>> #define QUIESCE_REJECT 2 /* Fail all calls with OPAL_BUSY */
>> diff --git a/hw/xive.c b/hw/xive.c
>> index 515f154d7894..c100c3bc84c7 100644
>> --- a/hw/xive.c
>> +++ b/hw/xive.c
>> @@ -1398,11 +1398,21 @@ static int64_t xive_ivc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>> return __xive_cache_scrub(x, xive_cache_ivc, block, idx, false, false);
>> }
>>
>> +static int64_t xive_vpc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>> +{
>> + return __xive_cache_scrub(x, xive_cache_vpc, block, idx, false, false);
>> +}
>> +
>> static int64_t xive_vpc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
>> {
>> return __xive_cache_scrub(x, xive_cache_vpc, block, idx, true, false);
>> }
>>
>> +static int64_t xive_eqc_scrub(struct xive *x, uint64_t block, uint64_t idx)
>> +{
>> + return __xive_cache_scrub(x, xive_cache_eqc, block, idx, false, false);
>> +}
>> +
>> static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
>> uint64_t block, uint64_t idx,
>> uint32_t start_dword, uint32_t dword_count,
>> @@ -4191,6 +4201,85 @@ static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
>> return rc;
>> }
>>
>> +static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
>> + uint32_t *out_qtoggle,
>> + uint32_t *out_qindex)
>> +{
>> + uint32_t blk, idx;
>> + struct xive *x;
>> + struct xive_eq *eq;
>> + int64_t rc;
>> +
>> + if (xive_mode != XIVE_MODE_EXPL)
>> + return OPAL_WRONG_STATE;
>> +
>> + if (!out_qtoggle || !out_qindex ||
>> + !xive_eq_for_target(vp, prio, &blk, &idx))
>> + return OPAL_PARAMETER;
>> +
>> + x = xive_from_vc_blk(blk);
>> + if (!x)
>> + return OPAL_PARAMETER;
>> +
>> + eq = xive_get_eq(x, idx);
>> + if (!eq)
>> + return OPAL_PARAMETER;
>> +
>> + /* Scrub the queue */
>> + lock(&x->lock);
>> + rc = xive_eqc_scrub(x, blk, idx);
>> + unlock(&x->lock);
>> + if (rc)
>> + return rc;
>> +
>> + /* We don't do disable queues */
>> + if (!(eq->w0 & EQ_W0_VALID))
>> + return OPAL_WRONG_STATE;
>> +
>> + *out_qtoggle = GETFIELD(EQ_W1_GENERATION, eq->w1);
>> + *out_qindex = GETFIELD(EQ_W1_PAGE_OFF, eq->w1);
>> +
>> + return OPAL_SUCCESS;
>> +}
>> +
>> +static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
>> + uint32_t qtoggle, uint32_t qindex)
>> +{
>> + uint32_t blk, idx;
>> + struct xive *x;
>> + struct xive_eq *eq, new_eq;
>> + int64_t rc;
>> +
>> + if (xive_mode != XIVE_MODE_EXPL)
>> + return OPAL_WRONG_STATE;
>> +
>> + if (!xive_eq_for_target(vp, prio, &blk, &idx))
>> + return OPAL_PARAMETER;
>> +
>> + x = xive_from_vc_blk(blk);
>> + if (!x)
>> + return OPAL_PARAMETER;
>> +
>> + eq = xive_get_eq(x, idx);
>> + if (!eq)
>> + return OPAL_PARAMETER;
>> +
>> + /* We don't do disable queues */
>> + if (!(eq->w0 & EQ_W0_VALID))
>> + return OPAL_WRONG_STATE;
>> +
>> + new_eq = *eq;
>> +
>> + new_eq.w1 = SETFIELD(EQ_W1_GENERATION, new_eq.w1, qtoggle);
>> + new_eq.w1 = SETFIELD(EQ_W1_PAGE_OFF, new_eq.w1, qindex);
>> +
>> + lock(&x->lock);
>> + rc = xive_eqc_cache_update(x, blk, idx, 0, 4, &new_eq, false, false);
>> + unlock(&x->lock);
>> +
>> + return rc;
>> +}
>> +
>> static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
>> {
>> struct proc_chip *c = get_chip(chip_id);
>> @@ -4436,6 +4525,48 @@ bail:
>> return rc;
>> }
>>
>> +static int64_t opal_xive_get_vp_state(uint64_t vp_id, uint64_t *out_state)
>> +{
>> + struct xive *x;
>> + struct xive_vp *vp;
>> + uint32_t blk, idx;
>> + int64_t rc;
>> + bool group;
>> +
>> + if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
>> + return OPAL_PARAMETER;
>> + if (group)
>> + return OPAL_PARAMETER;
>> + x = xive_from_pc_blk(blk);
>> + if (!x)
>> + return OPAL_PARAMETER;
>> + vp = xive_get_vp(x, idx);
>> + if (!vp)
>> + return OPAL_PARAMETER;
>> +
>> + /* Scrub the vp */
>> + lock(&x->lock);
>> + rc = xive_vpc_scrub(x, blk, idx);
>> + unlock(&x->lock);
>> + if (rc)
>> + return rc;
>> +
>> + if (!(vp->w0 & VP_W0_VALID))
>> + return OPAL_WRONG_STATE;
>> +
>> + /*
>> + * Return word4 and word5 which contain the saved HW thread
>> + * context. The IPB register is all we care for now on P9.
>> + */
>> + *out_state = (((uint64_t)vp->w4) << 32) | vp->w5;
>> +
>> + return OPAL_SUCCESS;
>> +}
>> +
>> +/*
>> + * TODO: P9 does not need to set the VP state but P10 might need to.
>> + */
>> +
>> static void xive_cleanup_cpu_tima(struct cpu_thread *c)
>> {
>> struct xive_cpu_state *xs = c->xstate;
>> @@ -5357,5 +5488,8 @@ void init_xive(void)
>> opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
>> opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
>> opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
>> + opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
>> + opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
>> + opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
>> }
>>
>> diff --git a/doc/xive.rst b/doc/xive.rst
>> index 0997c7224e11..cf5444336378 100644
>> --- a/doc/xive.rst
>> +++ b/doc/xive.rst
>> @@ -790,3 +790,53 @@ state information about the XIVE.
>> - XIVE_DUMP_EMU: Dump the state of the XICS emulation for a thread
>> "id" is the PIR value of the thread
>>
>> +
>> +OPAL_XIVE_GET_QUEUE_STATE
>> +^^^^^^^^^^^^^^^^^^^^^^^^^
>> +.. code-block:: c
>> +
>> + int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
>> + uint32_t *out_qtoggle,
>> + uint32_t *out_qindex);
>> +
>> +This call saves the queue toggle bit and index. This must be called on
>> +an enabled queue.
>> +
>> +* vp, prio: The target queue
>> +
>> +* out_qtoggle: toggle bit of the queue
>> +
>> +* out_qindex: index of the queue
>> +
>> +
>> +OPAL_XIVE_SET_QUEUE_STATE
>> +^^^^^^^^^^^^^^^^^^^^^^^^^
>> +.. code-block:: c
>> +
>> + int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
>> + uint32_t qtoggle,
>> + uint32_t qindex);
>> +
>> +This call restores the queue toggle bit and index that was previously
>> +saved by a call to opal_xive_get_queue_state(). This must be called on
>> +an enabled queue.
>> +
>> +* vp, prio: The target queue
>> +
>> +* qtoggle: toggle bit of the queue
>> +
>> +* qindex: index of the queue
>> +
>> +OPAL_XIVE_GET_VP_STATE
>> +^^^^^^^^^^^^^^^^^^^^^^
>> +.. code-block:: c
>> +
>> + int64_t opal_xive_get_vp_state(uint64_t vp_id,
>> + uint64_t *out_state);
>> +
>> +This call saves the VP HW state in "out_state". The format matches the
>> +XIVE NVT word 4 and word 5. This must be called on an enabled VP.
>> +
>> +* vp_id: The target VP
>> +
>> +* out_state: Location where the state is to be stored
>> --
>> 2.20.1
>>
>> _______________________________________________
>> Skiboot mailing list
>> Skiboot at lists.ozlabs.org
>> https://lists.ozlabs.org/listinfo/skiboot
More information about the Skiboot
mailing list