[Skiboot] [PATCH v2 5/8] npu2: Refactor NPU OPAL calls
Alexey Kardashevskiy
aik at ozlabs.ru
Mon Jul 15 15:53:21 AEST 2019
On 09/07/2019 07:07, Reza Arbab wrote:
> Abstract the OPAL entry points for npu2, moving them to a new file. This
> prepares us to add parallel npu3 versions of the same APIs.
>
> No functional change.
>
> Signed-off-by: Reza Arbab <arbab at linux.ibm.com>
> Reviewed-by: Andrew Donnellan <ajd at linux.ibm.com>
> ---
> hw/Makefile.inc | 1 +
> hw/npu-opal.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> hw/npu2.c | 149 ++++++--------------------------------------------
> include/npu2.h | 8 +++
> 4 files changed, 191 insertions(+), 132 deletions(-)
> create mode 100644 hw/npu-opal.c
>
> diff --git a/hw/Makefile.inc b/hw/Makefile.inc
> index 0c8a40367769..9c289189b20d 100644
> --- a/hw/Makefile.inc
> +++ b/hw/Makefile.inc
> @@ -8,6 +8,7 @@ HW_OBJS += dts.o lpc-rtc.o npu.o npu-hw-procedures.o xive.o phb4.o
> HW_OBJS += fake-nvram.o lpc-mbox.o npu2.o npu2-hw-procedures.o
> HW_OBJS += npu2-common.o npu2-opencapi.o phys-map.o sbe-p9.o capp.o
> HW_OBJS += occ-sensor.o vas.o sbe-p8.o dio-p9.o lpc-port80h.o cache-p9.o
> +HW_OBJS += npu-opal.o
> HW=hw/built-in.a
>
> include $(SRC)/hw/fsp/Makefile.inc
> diff --git a/hw/npu-opal.c b/hw/npu-opal.c
> new file mode 100644
> index 000000000000..f106c732ad6a
> --- /dev/null
> +++ b/hw/npu-opal.c
> @@ -0,0 +1,165 @@
> +/* Copyright 2019 IBM Corp.
> + *
> + * Licensed under the Apache License, Version 2.0 (the "License");
> + * you may not use this file except in compliance with the License.
> + * You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> + * implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +
> +#include <skiboot.h>
> +#include <pci.h>
> +#include <phb4.h>
> +#include <npu2.h>
> +
> +static int64_t opal_npu_init_context(uint64_t phb_id, int pid __unused,
> + uint64_t msr, uint64_t bdf)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> +
> + if (!phb || phb->phb_type != phb_type_npu_v2)
This and the same chunk but with phb_type_npu_v3 in 7/8 suggest these
opal_npu_init_context/etc should go to phb_ops or even create an
phb_npu_ops and add callback there for every case where you do "phb_type
!= phb_type_npu_vX" no?
> + return OPAL_PARAMETER;
> +
> + return npu2_init_context(phb, msr, bdf);
> +}
> +opal_call(OPAL_NPU_INIT_CONTEXT, opal_npu_init_context, 4);
> +
> +static int64_t opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused,
> + uint64_t bdf)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> +
> + if (!phb || phb->phb_type != phb_type_npu_v2)
> + return OPAL_PARAMETER;
> +
> + return npu2_destroy_context(phb, bdf);
> +}
> +opal_call(OPAL_NPU_DESTROY_CONTEXT, opal_npu_destroy_context, 3);
> +
> +static int64_t opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
> + uint64_t lpcr)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> +
> + if (!phb || phb->phb_type != phb_type_npu_v2)
> + return OPAL_PARAMETER;
> +
> + return npu2_map_lpar(phb, bdf, lparid, lpcr);
> +}
> +opal_call(OPAL_NPU_MAP_LPAR, opal_npu_map_lpar, 4);
> +
> +static int npu_check_relaxed_ordering(struct phb *phb, struct pci_device *pd,
> + void *enable)
> +{
> + /*
> + * IBM PCIe bridge devices (ie. the root ports) can always allow relaxed
> + * ordering
> + */
> + if (pd->vdid == 0x04c11014)
Nit (I do realize this moved unchanged from elsewhere): I am a bit
surprised not to see vendor ids defined in include/pci-cfg.h. Do we know
for sure the device id won't change? Checking for IBM vendor id + root
port would be cleaner imho. Thanks,
> + pd->allow_relaxed_ordering = true;
> +
> + PCIDBG(phb, pd->bdfn, "Checking relaxed ordering config\n");
> + if (pd->allow_relaxed_ordering)
> + return 0;
> +
> + PCIDBG(phb, pd->bdfn, "Relaxed ordering not allowed\n");
> + *(bool *)enable = false;
> +
> + return 1;
> +}
> +
> +static int64_t npu_set_relaxed_order(uint32_t gcid, int pec, bool enable)
> +{
> + struct phb *phb;
> + int64_t rc;
> +
> + for_each_phb(phb) {
> + if (phb->phb_type != phb_type_npu_v2)
> + continue;
> +
> + rc = npu2_set_relaxed_order(phb, gcid, pec, enable);
> + if (rc)
> + return rc;
> + }
> +
> + return OPAL_SUCCESS;
> +}
> +
> +static int64_t opal_npu_set_relaxed_order(uint64_t phb_id, uint16_t bdfn,
> + bool request_enabled)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> + struct phb4 *phb4;
> + uint32_t chip_id, pec;
> + struct pci_device *pd;
> + bool enable = true;
> +
> + if (!phb || phb->phb_type != phb_type_pcie_v4)
> + return OPAL_PARAMETER;
> +
> + phb4 = phb_to_phb4(phb);
> + pec = phb4->pec;
> + chip_id = phb4->chip_id;
> +
> + if (chip_id & ~0x1b)
> + return OPAL_PARAMETER;
> +
> + pd = pci_find_dev(phb, bdfn);
> + if (!pd)
> + return OPAL_PARAMETER;
> +
> + /*
> + * Not changing state, so no need to rescan PHB devices to determine if
> + * we need to enable/disable it
> + */
> + if (pd->allow_relaxed_ordering == request_enabled)
> + return OPAL_SUCCESS;
> +
> + pd->allow_relaxed_ordering = request_enabled;
> +
> + /*
> + * Walk all devices on this PHB to ensure they all support relaxed
> + * ordering
> + */
> + pci_walk_dev(phb, NULL, npu_check_relaxed_ordering, &enable);
> +
> + if (request_enabled && !enable) {
> + /*
> + * Not all devices on this PHB support relaxed-ordering
> + * mode so we can't enable it as requested
> + */
> + prlog(PR_INFO, "Cannot set relaxed ordering for PEC %d on chip %d\n",
> + pec, chip_id);
> + return OPAL_CONSTRAINED;
> + }
> +
> + if (npu_set_relaxed_order(chip_id, pec, request_enabled)) {
> + npu_set_relaxed_order(chip_id, pec, false);
> + return OPAL_RESOURCE;
> + }
> +
> + phb4->ro_state = request_enabled;
> + return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_NPU_SET_RELAXED_ORDER, opal_npu_set_relaxed_order, 3);
> +
> +static int64_t opal_npu_get_relaxed_order(uint64_t phb_id,
> + uint16_t bdfn __unused)
> +{
> + struct phb *phb = pci_get_phb(phb_id);
> + struct phb4 *phb4;
> +
> + if (!phb || phb->phb_type != phb_type_pcie_v4)
> + return OPAL_PARAMETER;
> +
> + phb4 = phb_to_phb4(phb);
> + return phb4->ro_state;
> +}
> +opal_call(OPAL_NPU_GET_RELAXED_ORDER, opal_npu_get_relaxed_order, 2);
> diff --git a/hw/npu2.c b/hw/npu2.c
> index 23e6157aefdf..4471fb0b81b8 100644
> --- a/hw/npu2.c
> +++ b/hw/npu2.c
> @@ -1938,17 +1938,12 @@ static int npu_table_search(struct npu2 *p, uint64_t table_addr, int stride,
> * allocated.
> */
> #define NPU2_VALID_ATS_MSR_BITS (MSR_DR | MSR_HV | MSR_PR | MSR_SF)
> -static int64_t opal_npu_init_context(uint64_t phb_id, int pasid __unused,
> - uint64_t msr, uint64_t bdf)
> +int64_t npu2_init_context(struct phb *phb, uint64_t msr, uint64_t bdf)
> {
> - struct phb *phb = pci_get_phb(phb_id);
> struct npu2 *p;
> uint64_t xts_bdf, old_xts_bdf_pid, xts_bdf_pid;
> int id;
>
> - if (!phb || phb->phb_type != phb_type_npu_v2)
> - return OPAL_PARAMETER;
> -
> /*
> * MSR bits should be masked by the caller to allow for future
> * expansion if required.
> @@ -2026,19 +2021,13 @@ out:
> unlock(&p->lock);
> return id;
> }
> -opal_call(OPAL_NPU_INIT_CONTEXT, opal_npu_init_context, 4);
>
> -static int opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused,
> - uint64_t bdf)
> +int64_t npu2_destroy_context(struct phb *phb, uint64_t bdf)
> {
> - struct phb *phb = pci_get_phb(phb_id);
> struct npu2 *p;
> uint64_t xts_bdf;
> int rc = OPAL_PARAMETER, id;
>
> - if (!phb || phb->phb_type != phb_type_npu_v2)
> - return OPAL_PARAMETER;
> -
> p = phb_to_npu2_nvlink(phb);
> lock(&p->lock);
>
> @@ -2067,15 +2056,13 @@ static int opal_npu_destroy_context(uint64_t phb_id, uint64_t pid __unused,
> unlock(&p->lock);
> return rc;
> }
> -opal_call(OPAL_NPU_DESTROY_CONTEXT, opal_npu_destroy_context, 3);
>
> /*
> * Map the given virtual bdf to lparid with given lpcr.
> */
> -static int opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
> - uint64_t lpcr)
> +int64_t npu2_map_lpar(struct phb *phb, uint64_t bdf, uint64_t lparid,
> + uint64_t lpcr)
> {
> - struct phb *phb = pci_get_phb(phb_id);
> struct npu2 *p;
> struct npu2_dev *ndev = NULL;
> uint64_t xts_bdf_lpar, atsd_lpar, rc = OPAL_SUCCESS;
> @@ -2088,9 +2075,6 @@ static int opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
> NPU2_XTS_MMIO_ATSD6_LPARID, NPU2_XTS_MMIO_ATSD7_LPARID
> };
>
> - if (!phb || phb->phb_type != phb_type_npu_v2)
> - return OPAL_PARAMETER;
> -
> if (lpcr)
> /* The LPCR bits are only required for hash based ATS,
> * which we don't currently support but may need to in
> @@ -2173,7 +2157,6 @@ out:
> unlock(&p->lock);
> return rc;
> }
> -opal_call(OPAL_NPU_MAP_LPAR, opal_npu_map_lpar, 4);
>
> static inline uint32_t npu2_relaxed_ordering_source_grpchp(uint32_t gcid)
> {
> @@ -2319,123 +2302,25 @@ static void npu2_disable_relaxed_ordering(struct npu2_dev *ndev, uint32_t gcid,
> * relaxed ordering partially enabled if there are insufficient HW resources to
> * enable it on all links.
> */
> -static int npu2_set_relaxed_ordering(uint32_t gcid, int pec, bool enable)
> +int64_t npu2_set_relaxed_order(struct phb *phb, uint32_t gcid, int pec,
> + bool enable)
> {
> - int rc = OPAL_SUCCESS;
> - struct phb *phb;
> - struct npu2 *npu;
> + struct npu2 *npu = phb_to_npu2_nvlink(phb);
> struct npu2_dev *ndev;
> + int64_t rc = OPAL_SUCCESS;
>
> - for_each_phb(phb) {
> - if (phb->phb_type != phb_type_npu_v2)
> - continue;
> + for (int i = 0; i < npu->total_devices; i++) {
> + ndev = &npu->devices[i];
> + if (enable)
> + rc = npu2_enable_relaxed_ordering(ndev, gcid, pec);
> + else
> + npu2_disable_relaxed_ordering(ndev, gcid, pec);
>
> - npu = phb_to_npu2_nvlink(phb);
> - for (int i = 0; i < npu->total_devices; i++) {
> - ndev = &npu->devices[i];
> - if (enable)
> - rc = npu2_enable_relaxed_ordering(ndev, gcid, pec);
> - else
> - npu2_disable_relaxed_ordering(ndev, gcid, pec);
> -
> - if (rc != OPAL_SUCCESS) {
> - NPU2DEVINF(ndev, "Insufficient resources to activate relaxed ordering mode\n");
> - return OPAL_RESOURCE;
> - }
> + if (rc != OPAL_SUCCESS) {
> + NPU2DEVINF(ndev, "Insufficient resources to activate relaxed ordering mode\n");
> + return OPAL_RESOURCE;
> }
> }
>
> return OPAL_SUCCESS;
> }
> -
> -static int npu2_check_relaxed_ordering(struct phb *phb __unused,
> - struct pci_device *pd, void *enable)
> -{
> - /*
> - * IBM PCIe bridge devices (ie. the root ports) can always allow relaxed
> - * ordering
> - */
> - if (pd->vdid == 0x04c11014)
> - pd->allow_relaxed_ordering = true;
> -
> - PCIDBG(phb, pd->bdfn, "Checking relaxed ordering config\n");
> - if (pd->allow_relaxed_ordering)
> - return 0;
> -
> - PCIDBG(phb, pd->bdfn, "Relaxed ordering not allowed\n");
> - *(bool *) enable = false;
> -
> - return 1;
> -}
> -
> -static int64_t opal_npu_set_relaxed_order(uint64_t phb_id, uint16_t bdfn,
> - bool request_enabled)
> -{
> - struct phb *phb = pci_get_phb(phb_id);
> - struct phb4 *phb4;
> - uint32_t chip_id, pec;
> - struct pci_device *pd;
> - bool enable = true;
> -
> - if (!phb || phb->phb_type != phb_type_pcie_v4)
> - return OPAL_PARAMETER;
> -
> - phb4 = phb_to_phb4(phb);
> - pec = phb4->pec;
> - chip_id = phb4->chip_id;
> -
> - if (npu2_relaxed_ordering_source_grpchp(chip_id) == OPAL_PARAMETER)
> - return OPAL_PARAMETER;
> -
> - pd = pci_find_dev(phb, bdfn);
> - if (!pd)
> - return OPAL_PARAMETER;
> -
> - /*
> - * Not changing state, so no need to rescan PHB devices to determine if
> - * we need to enable/disable it
> - */
> - if (pd->allow_relaxed_ordering == request_enabled)
> - return OPAL_SUCCESS;
> -
> - pd->allow_relaxed_ordering = request_enabled;
> -
> - /*
> - * Walk all devices on this PHB to ensure they all support relaxed
> - * ordering
> - */
> - pci_walk_dev(phb, NULL, npu2_check_relaxed_ordering, &enable);
> -
> - if (request_enabled && !enable) {
> - /*
> - * Not all devices on this PHB support relaxed-ordering
> - * mode so we can't enable it as requested
> - */
> - prlog(PR_INFO, "Cannot set relaxed ordering for PEC %d on chip %d\n",
> - pec, chip_id);
> - return OPAL_CONSTRAINED;
> - }
> -
> - if (npu2_set_relaxed_ordering(chip_id, pec, request_enabled) != OPAL_SUCCESS) {
> - npu2_set_relaxed_ordering(chip_id, pec, false);
> - return OPAL_RESOURCE;
> - }
> -
> - phb4->ro_state = request_enabled;
> - return OPAL_SUCCESS;
> -}
> -opal_call(OPAL_NPU_SET_RELAXED_ORDER, opal_npu_set_relaxed_order, 3);
> -
> -static int64_t opal_npu_get_relaxed_order(uint64_t phb_id,
> - uint16_t bdfn __unused)
> -{
> - struct phb *phb = pci_get_phb(phb_id);
> - struct phb4 *phb4;
> -
> - if (!phb || phb->phb_type != phb_type_pcie_v4)
> - return OPAL_PARAMETER;
> -
> - phb4 = phb_to_phb4(phb);
> - return phb4->ro_state;
> -}
> -opal_call(OPAL_NPU_GET_RELAXED_ORDER, opal_npu_get_relaxed_order, 2);
> diff --git a/include/npu2.h b/include/npu2.h
> index 5b2a436bda06..21a61712b5b9 100644
> --- a/include/npu2.h
> +++ b/include/npu2.h
> @@ -256,4 +256,12 @@ int64_t npu2_freeze_status(struct phb *phb __unused,
> uint16_t *pci_error_type __unused,
> uint16_t *severity __unused);
> void npu2_dump_scoms(int chip_id);
> +
> +int64_t npu2_init_context(struct phb *phb, uint64_t msr, uint64_t bdf);
> +int64_t npu2_destroy_context(struct phb *phb, uint64_t bdf);
> +int64_t npu2_map_lpar(struct phb *phb, uint64_t bdf, uint64_t lparid,
> + uint64_t lpcr);
> +int64_t npu2_set_relaxed_order(struct phb *phb, uint32_t gcid, int pec,
> + bool enable);
> +
> #endif /* __NPU2_H */
>
--
Alexey
More information about the Skiboot
mailing list