[Cbe-oss-dev] [RFC/PATCH] libspe2: use mapped mailbox registers to speed up mailbox communication

[Kazunori Asayama]

stenzel at de.ibm.com wrote:
> The following patch speeds up mailbox communication by
> accessing the mailbox registers directly via mapped problem
> state if the context was created with the SPE_MAP_PS flag.
> 
> Comments are appreciated.

A problem of this approach is that there is no guarantee that check of
mbox-stat and access to mbox are done atomically if both of syscall
and direct access are performed at the same time.

(snip)

> +static __inline__ int _base_spe_in_mbox_write_ps(spe_context_ptr_t spectx,
> +                        unsigned int *mbox_data, 
> +                        int count)
> +{
> +	volatile struct spe_spu_control_area *cntl_area =
> +        	spectx->base_private->cntl_mmap_base;
> +	int total = 0;
> +	unsigned int *aux;
> +
> +	_base_spe_context_lock(spectx, FD_WBOX);
> +	aux = mbox_data;
> +	while (total < count) {
> +		int space = (cntl_area->SPU_Mbox_Stat >> 8) & 0xFF;
> +		if (space) {
> +			cntl_area->SPU_In_Mbox = *aux++;
> +			total++;
> +		} else {
> +			break;
> +		}
> +	}
> +	_base_spe_context_unlock(spectx, FD_WBOX);
> +
> +	return total;
> +}
> +

Consider that one thread call the function above while another thread
is still calling write() for the same mbox. Such a situation seems
possible when two thread call spe_in_mbox_write() with
SPE_MBOX_ANY_BLOCKING at the same time.

>  int _base_spe_in_mbox_write(spe_context_ptr_t spectx, 
>                          unsigned int *mbox_data, 
>                          int count, 
> @@ -63,6 +115,7 @@ int _base_spe_in_mbox_write(spe_context_
>  {
>  	int rc;
>  	int total;
> +	unsigned int *aux;
>  
>  	if (mbox_data == NULL || count < 1){
>  		errno = EINVAL;
> @@ -72,27 +125,48 @@ int _base_spe_in_mbox_write(spe_context_
>  	switch (behavior_flag) {
>  	case SPE_MBOX_ALL_BLOCKING: // write all, even if blocking
>  		total = rc = 0;
> -		while (total < 4*count) {
> +
> +		// first try to write directly if we map the PS
> +		if (spectx->base_private->flags & SPE_MAP_PS) 
> +			total = _base_spe_in_mbox_write_ps(spectx, mbox_data, count);
> +
> +		// now write the remaining via spufs
> +		while (total < count) {
> +			aux = mbox_data + total;
>  			rc = write(open_if_closed(spectx,FD_WBOX, 0),
> -					(const char *)mbox_data + total, 4*count - total);
> +				  aux, 4*(count - total));
>  			if (rc == -1) {
>  				break;
>  			}
> -			total += rc;
> +			total += (rc/4);
>  		}
>  		break;
>  
>  	case  SPE_MBOX_ANY_BLOCKING: // write at least one, even if blocking
> -		total = rc = write(open_if_closed(spectx,FD_WBOX, 0), mbox_data, 4*count);
> +		total = rc = 0;
> +		// first try to write directly if we map the PS
> +		if (spectx->base_private->flags & SPE_MAP_PS) 
> +			total = _base_spe_in_mbox_write_ps(spectx, mbox_data, count);
> +		// if none was written write via spufs
> +		if (total == 0) {
> +			rc = write(open_if_closed(spectx,FD_WBOX, 0), mbox_data, 4*count);
> +			total = rc/4;
> +		}
>  		break;
>  
>  	case  SPE_MBOX_ANY_NONBLOCKING: // only write, if non blocking
> -		rc = write(open_if_closed(spectx,FD_WBOX_NB, 0), mbox_data, 4*count);
> -		if (rc == -1 && errno == EAGAIN) {
> -			rc = 0;
> -			errno = 0;
> +		total = rc = 0;
> +		// write directly if we map the PS else write via spufs
> +		if (spectx->base_private->flags & SPE_MAP_PS) {
> +			total = _base_spe_in_mbox_write_ps(spectx, mbox_data, count);
> +		} else { 
> +			rc = write(open_if_closed(spectx,FD_WBOX_NB, 0), mbox_data, 4*count);
> +			if (rc == -1 && errno == EAGAIN) {
> +				rc = 0;
> +				errno = 0;
> +			}
> +			total = rc/4;
>  		}
> -		total = rc;
>  		break;
>  
>  	default:
> @@ -105,16 +179,24 @@ int _base_spe_in_mbox_write(spe_context_
>  		return -1;
>  	}
>  
> -	return total / 4;
> +	return total;
>  }
>  

--
(ASAYAMA Kazunori
  (asayama at sm.sony.co.jp))
t