Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-27 13:22:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

habanalabs: abort waiting user threads upon error

Browse Source 
User should close the FD when being notified about an error, after
which a device reset takes place.

However, if the user has pending threads that wait for completions,
the device release won't be called and eventually the watchdog timeout
will expire, leading to hard reset and killing the user process.

To avoid it, abort such waiting threads right after the error
notification, and block following waiting operations.

Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>
This commit is contained in:
Tomer Tayar 2022-11-08 14:34:43 +02:00 committed by Oded Gabbay
parent cdacf3c000
commit ce259804d2
3 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
drivers/misc/habanalabs/common/command_submission.c
View File

@ -1117,6 +1117,27 @@ void hl_release_pending_user_interrupts(struct hl_device *hdev)
wake_pending_user_interrupt_threads(interrupt);
}
static void force_complete_cs(struct hl_device *hdev)
{
struct hl_cs *cs;
spin_lock(&hdev->cs_mirror_lock);
list_for_each_entry(cs, &hdev->cs_mirror_list, mirror_node) {
cs->fence->error = -EIO;
complete_all(&cs->fence->completion);
}
spin_unlock(&hdev->cs_mirror_lock);
}
void hl_abort_waitings_for_completion(struct hl_device *hdev)
{
force_complete_cs(hdev);
force_complete_multi_cs(hdev);
hl_release_pending_user_interrupts(hdev);
}
static void job_wq_completion(struct work_struct *work)
{
struct hl_cs_job *job = container_of(work, struct hl_cs_job,
@ -3489,14 +3510,15 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)
int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data)
{
struct hl_device *hdev = hpriv->hdev;
union hl_wait_cs_args *args = data;
u32 flags = args->in.flags;
int rc;
/* If the device is not operational, no point in waiting for any command submission or
* user interrupt
/* If the device is not operational, or if an error has happened and user should release the
* device, there is no point in waiting for any command submission or user interrupt.
*/
if (!hl_device_operational(hpriv->hdev, NULL))
if (!hl_device_operational(hpriv->hdev, NULL) || hdev->reset_info.watchdog_active)
return -EBUSY;
if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)

2
drivers/misc/habanalabs/common/device.c
View File

@ -1865,6 +1865,8 @@ out:
hl_ctx_put(ctx);
hl_abort_waitings_for_completion(hdev);
return 0;
device_reset:

1
drivers/misc/habanalabs/common/habanalabs.h
View File

@ -3791,6 +3791,7 @@ void hl_dec_fini(struct hl_device *hdev);
void hl_dec_ctx_fini(struct hl_ctx *ctx);
void hl_release_pending_user_interrupts(struct hl_device *hdev);
void hl_abort_waitings_for_completion(struct hl_device *hdev);
int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig);