[Skiboot] [PATCH v2 5/5] npu2: Add support for relaxed-ordering mode

[Stewart Smith]

Reza Arbab <arbab at linux.ibm.com> writes:
> Some device drivers support out of order access to GPU memory. This does
> not affect the CPU view of memory but it does affect the GPU view of
> memory. It should only be enabled once the GPU driver has requested it
> to be enabled.
>
> Current hardware only allows it to be enabled per PCIe root port,
> therefore all devices on a given PCIe root port must be explicitly
> enabled for relaxed-ordering prior to it actually being enabled.

A bit about the new API calls would be good here, and their behaviour in
supporting any future hardware.
> +
> +static int64_t opal_npu_set_relaxed_order(uint64_t phb_id, uint16_t bdfn,
> +					  bool request_enabled)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	struct phb4 *phb4;
> +	uint32_t chip_id, pec;
> +	struct pci_device *pd;
> +	bool enable = true;
> +
> +	if (!phb || phb->phb_type != phb_type_pcie_v4)
> +		return OPAL_PARAMETER;
> +
> +	phb4 = phb_to_phb4(phb);
> +	pec = phb4->pec;
> +	chip_id = phb4->chip_id;
> +
> +	if (npu2_relaxed_ordering_source_grpchp(chip_id) == OPAL_PARAMETER)
> +		return OPAL_PARAMETER;
> +
> +	pd = pci_find_dev(phb, bdfn);
> +	if (!pd)
> +		return OPAL_PARAMETER;
> +
> +	/*
> +	 * Not changing state, so no need to rescan PHB devices to determine if
> +	 * we need to enable/disable it
> +	 */
> +	if (pd->allow_relaxed_ordering == request_enabled)
> +		return OPAL_SUCCESS;
> +
> +	pd->allow_relaxed_ordering = request_enabled;
> +
> +	/*
> +	 * Walk all devices on this PHB to ensure they all support relaxed
> +	 * ordering
> +	 */
> +	pci_walk_dev(phb, NULL, npu2_check_relaxed_ordering, &enable);
> +
> +	if (request_enabled && !enable) {
> +		/*
> +		 * Not all devices on this PHB support relaxed-ordering
> +		 * mode so we can't enable it as requested
> +		 */
> +		prlog(PR_INFO, "Cannot set relaxed ordering for PEC %d on chip %d\n",
> +		      pec, chip_id);
> +		return OPAL_CONSTRAINED;
> +	}
> +
> +	if (npu2_set_relaxed_ordering(chip_id, pec, request_enabled) != OPAL_SUCCESS) {
> +		npu2_set_relaxed_ordering(chip_id, pec, false);
> +		return OPAL_RESOURCE;
> +	}
> +
> +	phb4->ro_state = request_enabled;
> +	return OPAL_SUCCESS;
> +}
> +opal_call(OPAL_NPU_SET_RELAXED_ORDER, opal_npu_set_relaxed_order, 3);
> +
> +static int64_t opal_npu_get_relaxed_order(uint64_t phb_id,
> +					  uint16_t bdfn __unused)
> +{
> +	struct phb *phb = pci_get_phb(phb_id);
> +	struct phb4 *phb4;
> +
> +	if (!phb || phb->phb_type != phb_type_pcie_v4)
> +		return OPAL_PARAMETER;
> +
> +	phb4 = phb_to_phb4(phb);
> +	return phb4->ro_state;
> +}
> +opal_call(OPAL_NPU_GET_RELAXED_ORDER, opal_npu_get_relaxed_order, 2);

You'll need to add things to doc/opal-api/ describing the new API calls
and how they interact with the existing ones.

-- 
Stewart Smith
OPAL Architect, IBM.