From 464fd6419c68bc6b1697e02f46b6d3dd57dfed28 Mon Sep 17 00:00:00 2001 From: Michael Cyr Date: Tue, 16 May 2017 17:49:21 -0500 Subject: [PATCH] ibmvscsis: Enable Logical Partition Migration Support Changes to support a new mechanism from phyp to better synchronize the logical partition migration (LPM) of the client partition. This includes a new VIOCTL to register that we support this new functionality, and 2 new Transport Event types, and finally another new VIOCTL to let phyp know once we're ready for the Suspend. Signed-off-by: Michael Cyr Signed-off-by: Bryant G. Ly Signed-off-by: Nicholas Bellinger --- drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c | 148 +++++++++++++++++++++-- drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h | 25 +++- drivers/scsi/ibmvscsi_tgt/libsrp.h | 5 +- 3 files changed, 162 insertions(+), 16 deletions(-) diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c index abf6026645dd..35710524d059 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c @@ -155,6 +155,9 @@ static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi) qrc = h_free_crq(vscsi->dds.unit_id); switch (qrc) { case H_SUCCESS: + spin_lock_bh(&vscsi->intr_lock); + vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS; + spin_unlock_bh(&vscsi->intr_lock); break; case H_HARDWARE: @@ -422,6 +425,9 @@ static void ibmvscsis_disconnect(struct work_struct *work) new_state = vscsi->new_state; vscsi->new_state = 0; + vscsi->flags |= DISCONNECT_SCHEDULED; + vscsi->flags &= ~SCHEDULE_DISCONNECT; + pr_debug("disconnect: flags 0x%x, state 0x%hx\n", vscsi->flags, vscsi->state); @@ -802,6 +808,13 @@ static long ibmvscsis_establish_new_q(struct scsi_info *vscsi) long rc = ADAPT_SUCCESS; uint format; + rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000, + 0, 0, 0, 0); + if (rc == H_SUCCESS) + vscsi->flags |= PREP_FOR_SUSPEND_ENABLED; + else if (rc != H_NOT_FOUND) + pr_err("Error from Enable Prepare for Suspend: %ld\n", rc); + vscsi->flags &= PRESERVE_FLAG_FIELDS; vscsi->rsp_q_timer.timer_pops = 0; vscsi->debit = 0; @@ -950,6 +963,63 @@ static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi, } } +/** + * ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL + * @vscsi: Pointer to our adapter structure + * @idle: Indicates whether we were called from adapter_idle. This + * is important to know if we need to do a disconnect, since if + * we're called from adapter_idle, we're still processing the + * current disconnect, so we can't just call post_disconnect. + * + * This function is called when the adapter is idle when phyp has sent + * us a Prepare for Suspend Transport Event. + * + * EXECUTION ENVIRONMENT: + * Process or interrupt environment called with interrupt lock held + */ +static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle) +{ + long rc = 0; + struct viosrp_crq *crq; + + /* See if there is a Resume event in the queue */ + crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index; + + pr_debug("ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n", + vscsi->flags, vscsi->state, (int)crq->valid); + + if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) { + rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0, + 0, 0); + if (rc) { + pr_err("Ready for Suspend Vioctl failed: %ld\n", rc); + rc = 0; + } + } else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) && + (vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) || + ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) || + (crq->format != RESUME_FROM_SUSP)))) { + if (idle) { + vscsi->state = ERR_DISCONNECT_RECONNECT; + ibmvscsis_reset_queue(vscsi); + rc = -1; + } else if (vscsi->state == CONNECTED) { + ibmvscsis_post_disconnect(vscsi, + ERR_DISCONNECT_RECONNECT, 0); + } + + vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE; + + if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) || + (crq->format != RESUME_FROM_SUSP))) + pr_err("Invalid element in CRQ after Prepare for Suspend"); + } + + vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED); + + return rc; +} + /** * ibmvscsis_trans_event() - Handle a Transport Event * @vscsi: Pointer to our adapter structure @@ -974,18 +1044,8 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi, case PARTNER_FAILED: case PARTNER_DEREGISTER: ibmvscsis_delete_client_info(vscsi, true); - break; - - default: - rc = ERROR; - dev_err(&vscsi->dev, "trans_event: invalid format %d\n", - (uint)crq->format); - ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, - RESPONSE_Q_DOWN); - break; - } - - if (rc == ADAPT_SUCCESS) { + if (crq->format == MIGRATED) + vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE; switch (vscsi->state) { case NO_QUEUE: case ERR_DISCONNECTED: @@ -1034,6 +1094,60 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi, vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT); break; } + break; + + case PREPARE_FOR_SUSPEND: + pr_debug("Prep for Suspend, crq status = 0x%x\n", + (int)crq->status); + switch (vscsi->state) { + case ERR_DISCONNECTED: + case WAIT_CONNECTION: + case CONNECTED: + ibmvscsis_ready_for_suspend(vscsi, false); + break; + case SRP_PROCESSING: + vscsi->resume_state = vscsi->state; + vscsi->flags |= PREP_FOR_SUSPEND_PENDING; + if (crq->status == CRQ_ENTRY_OVERWRITTEN) + vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE; + ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0); + break; + case NO_QUEUE: + case UNDEFINED: + case UNCONFIGURING: + case WAIT_ENABLED: + case ERR_DISCONNECT: + case ERR_DISCONNECT_RECONNECT: + case WAIT_IDLE: + pr_err("Invalid state for Prepare for Suspend Trans Event: 0x%x\n", + vscsi->state); + break; + } + break; + + case RESUME_FROM_SUSP: + pr_debug("Resume from Suspend, crq status = 0x%x\n", + (int)crq->status); + if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) { + vscsi->flags |= PREP_FOR_SUSPEND_ABORTED; + } else { + if ((crq->status == CRQ_ENTRY_OVERWRITTEN) || + (vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) { + ibmvscsis_post_disconnect(vscsi, + ERR_DISCONNECT_RECONNECT, + 0); + vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE; + } + } + break; + + default: + rc = ERROR; + dev_err(&vscsi->dev, "trans_event: invalid format %d\n", + (uint)crq->format); + ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, + RESPONSE_Q_DOWN); + break; } rc = vscsi->flags & SCHEDULE_DISCONNECT; @@ -1201,6 +1315,7 @@ static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi) static void ibmvscsis_adapter_idle(struct scsi_info *vscsi) { int free_qs = false; + long rc = 0; pr_debug("adapter_idle: flags 0x%x, state 0x%hx\n", vscsi->flags, vscsi->state); @@ -1240,7 +1355,14 @@ static void ibmvscsis_adapter_idle(struct scsi_info *vscsi) vscsi->rsp_q_timer.timer_pops = 0; vscsi->debit = 0; vscsi->credit = 0; - if (vscsi->flags & TRANS_EVENT) { + if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) { + vscsi->state = vscsi->resume_state; + vscsi->resume_state = 0; + rc = ibmvscsis_ready_for_suspend(vscsi, true); + vscsi->flags &= ~DISCONNECT_SCHEDULED; + if (rc) + break; + } else if (vscsi->flags & TRANS_EVENT) { vscsi->state = WAIT_CONNECTION; vscsi->flags &= PRESERVE_FLAG_FIELDS; } else { diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h index b4391a8de456..cc96c2731134 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h @@ -262,6 +262,14 @@ struct scsi_info { #define DISCONNECT_SCHEDULED 0x00800 /* remove function is sleeping */ #define CFG_SLEEPING 0x01000 + /* Register for Prepare for Suspend Transport Events */ +#define PREP_FOR_SUSPEND_ENABLED 0x02000 + /* Prepare for Suspend event sent */ +#define PREP_FOR_SUSPEND_PENDING 0x04000 + /* Resume from Suspend event sent */ +#define PREP_FOR_SUSPEND_ABORTED 0x08000 + /* Prepare for Suspend event overwrote another CRQ entry */ +#define PREP_FOR_SUSPEND_OVERWRITE 0x10000 u32 flags; /* adapter lock */ spinlock_t intr_lock; @@ -272,6 +280,7 @@ struct scsi_info { /* used in crq, to tag what iu the response is for */ u64 empty_iu_tag; uint new_state; + uint resume_state; /* control block for the response queue timer */ struct timer_cb rsp_q_timer; /* keep last client to enable proper accounting */ @@ -324,8 +333,13 @@ struct scsi_info { #define TARGET_STOP(VSCSI) (long)(((VSCSI)->state & DONT_PROCESS_STATE) | \ ((VSCSI)->flags & BLOCK)) +#define PREP_FOR_SUSPEND_FLAGS (PREP_FOR_SUSPEND_ENABLED | \ + PREP_FOR_SUSPEND_PENDING | \ + PREP_FOR_SUSPEND_ABORTED | \ + PREP_FOR_SUSPEND_OVERWRITE) + /* flag bit that are not reset during disconnect */ -#define PRESERVE_FLAG_FIELDS 0 +#define PRESERVE_FLAG_FIELDS (PREP_FOR_SUSPEND_FLAGS) #define vio_iu(IUE) ((union viosrp_iu *)((IUE)->sbuf->buf)) @@ -333,8 +347,15 @@ struct scsi_info { #define WRITE_CMD(cdb) (((cdb)[0] & 0x1F) == 0xA) #ifndef H_GET_PARTNER_INFO -#define H_GET_PARTNER_INFO 0x0000000000000008LL +#define H_GET_PARTNER_INFO 0x0000000000000008LL #endif +#ifndef H_ENABLE_PREPARE_FOR_SUSPEND +#define H_ENABLE_PREPARE_FOR_SUSPEND 0x000000000000001DLL +#endif +#ifndef H_READY_FOR_SUSPEND +#define H_READY_FOR_SUSPEND 0x000000000000001ELL +#endif + #define h_copy_rdma(l, sa, sb, da, db) \ plpar_hcall_norets(H_COPY_RDMA, l, sa, sb, da, db) diff --git a/drivers/scsi/ibmvscsi_tgt/libsrp.h b/drivers/scsi/ibmvscsi_tgt/libsrp.h index 4696f331453e..9fec55b36322 100644 --- a/drivers/scsi/ibmvscsi_tgt/libsrp.h +++ b/drivers/scsi/ibmvscsi_tgt/libsrp.h @@ -30,10 +30,13 @@ enum srp_trans_event { UNUSED_FORMAT = 0, PARTNER_FAILED = 1, PARTNER_DEREGISTER = 2, - MIGRATED = 6 + MIGRATED = 6, + PREPARE_FOR_SUSPEND = 9, + RESUME_FROM_SUSP = 0xA }; enum srp_status { + CRQ_ENTRY_OVERWRITTEN = 0x20, HEADER_DESCRIPTOR = 0xF1, PING = 0xF5, PING_RESPONSE = 0xF6