accel/habanalabs: dump the EQ entries headers on EQ heartbeat failure

Add a dump of the EQ entries headers upon a EQ heartbeat failure.

Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Ofir Bitton <obitton@habana.ai>
Signed-off-by: Ofir Bitton <obitton@habana.ai>
This commit is contained in:
Tomer Tayar 2024-04-16 19:01:42 +03:00 committed by Ofir Bitton
parent 795f93e650
commit c4548eee53
4 changed files with 31 additions and 0 deletions

View File

@ -1082,6 +1082,8 @@ static bool hl_device_eq_heartbeat_received(struct hl_device *hdev)
atomic_read(&hdev->kernel_queues[cpu_q_id].ci),
atomic_read(&hdev->kernel_queues[cpu_q_id].ci) & pq_pi_mask);
hl_eq_dump(hdev, &hdev->event_queue);
return false;
}

View File

@ -3754,6 +3754,7 @@ int hl_eq_init(struct hl_device *hdev, struct hl_eq *q);
void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q);
void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q);
void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q);
void hl_eq_dump(struct hl_device *hdev, struct hl_eq *q);
irqreturn_t hl_irq_handler_cq(int irq, void *arg);
irqreturn_t hl_irq_handler_eq(int irq, void *arg);
irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg);

View File

@ -697,3 +697,28 @@ void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
memset(q->kernel_address, 0, q->size);
}
void hl_eq_dump(struct hl_device *hdev, struct hl_eq *q)
{
u32 eq_length, eqe_size, ctl, ready, mode, type, index;
struct hl_eq_header *hdr;
u8 *ptr;
int i;
eq_length = HL_EQ_LENGTH;
eqe_size = q->size / HL_EQ_LENGTH;
dev_info(hdev->dev, "Contents of EQ entries headers:\n");
for (i = 0, ptr = q->kernel_address ; i < eq_length ; ++i, ptr += eqe_size) {
hdr = (struct hl_eq_header *) ptr;
ctl = le32_to_cpu(hdr->ctl);
ready = FIELD_GET(EQ_CTL_READY_MASK, ctl);
mode = FIELD_GET(EQ_CTL_EVENT_MODE_MASK, ctl);
type = FIELD_GET(EQ_CTL_EVENT_TYPE_MASK, ctl);
index = FIELD_GET(EQ_CTL_INDEX_MASK, ctl);
dev_info(hdev->dev, "%02u: %#010x [ready: %u, mode %u, type %04u, index %05u]\n",
i, ctl, ready, mode, type, index);
}
}

View File

@ -397,6 +397,9 @@ struct hl_eq_entry {
#define EQ_CTL_READY_SHIFT 31
#define EQ_CTL_READY_MASK 0x80000000
#define EQ_CTL_EVENT_MODE_SHIFT 28
#define EQ_CTL_EVENT_MODE_MASK 0x70000000
#define EQ_CTL_EVENT_TYPE_SHIFT 16
#define EQ_CTL_EVENT_TYPE_MASK 0x0FFF0000