Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-28 13:51:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

net/smc: use correct vlan gid of RoCE device

Browse Source 
SMC code uses the base gid for VLAN traffic. The gids exchanged in
the CLC handshake and the gid index used for the QP have to switch
from the base gid to the appropriate vlan gid.

When searching for a matching IB device port for a certain vlan
device, it does not make sense to return an IB device port, which
is not enabled for the used vlan_id. Add another check whether a
vlan gid exists for a certain IB device port.

Signed-off-by: Ursula Braun <ubraun@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Ursula Braun 2018-07-25 16:35:31 +02:00 committed by David S. Miller
parent 947541f36c
commit 7005ada68d
12 changed files with 93 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
net/smc/af_smc.c
View File

@ -370,8 +370,7 @@ static int smc_clnt_conf_first_link(struct smc_sock *smc)
/* send add link reject message, only one link supported for now */
rc = smc_llc_send_add_link(link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_RESP);
link->gid, SMC_LLC_RESP);
if (rc < 0)
return SMC_CLC_DECL_TCL;
@ -469,7 +468,7 @@ static int smc_connect_abort(struct smc_sock *smc, int reason_code,
/* check if there is a rdma device available for this connection. */
/* called for connect and listen */
static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
u8 *ibport)
u8 *ibport, unsigned short vlan_id, u8 gid[])
{
int reason_code = 0;
@ -477,7 +476,8 @@ static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
* within same PNETID that also contains the ethernet device
* used for the internal TCP socket
*/
smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport);
smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,
gid);
if (!(*ibdev))
reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
@ -523,12 +523,12 @@ static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
static int smc_connect_clc(struct smc_sock *smc, int smc_type,
struct smc_clc_msg_accept_confirm *aclc,
struct smc_ib_device *ibdev, u8 ibport,
struct smcd_dev *ismdev)
u8 gid[], struct smcd_dev *ismdev)
{
int rc = 0;
/* do inband token exchange */
rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, ismdev);
rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);
if (rc)
return rc;
/* receive SMC Accept CLC message */
@ -650,6 +650,7 @@ static int __smc_connect(struct smc_sock *smc)
struct smc_clc_msg_accept_confirm aclc;
struct smc_ib_device *ibdev;
struct smcd_dev *ismdev;
u8 gid[SMC_GID_SIZE];
unsigned short vlan;
int smc_type;
int rc = 0;
@ -681,7 +682,7 @@ static int __smc_connect(struct smc_sock *smc)
}
/* check if there is a rdma device available */
if (!smc_check_rdma(smc, &ibdev, &ibport)) {
if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {
/* RDMA is supported for this connection */
rdma_supported = true;
if (ism_supported)
@ -695,7 +696,7 @@ static int __smc_connect(struct smc_sock *smc)
return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
/* perform CLC handshake */
rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, ismdev);
rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);
if (rc) {
smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
return smc_connect_decline_fallback(smc, rc);
@ -970,8 +971,7 @@ static int smc_serv_conf_first_link(struct smc_sock *smc)
/* send ADD LINK request to client over the RoCE fabric */
rc = smc_llc_send_add_link(link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_REQ);
link->gid, SMC_LLC_REQ);
if (rc < 0)
return SMC_CLC_DECL_TCL;
@ -1193,6 +1193,7 @@ static void smc_listen_work(struct work_struct *work)
struct smcd_dev *ismdev;
u8 buf[SMC_CLC_MAX_LEN];
int local_contact = 0;
unsigned short vlan;
int reason_code = 0;
int rc = 0;
u8 ibport;
@ -1241,7 +1242,8 @@ static void smc_listen_work(struct work_struct *work)
/* check if RDMA is available */
if (!ism_supported &&
((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||
smc_check_rdma(new_smc, &ibdev, &ibport) ||
smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||
smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||
smc_listen_rdma_check(new_smc, pclc) ||
smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
&local_contact) ||

10
net/smc/smc_clc.c
View File

@ -378,7 +378,7 @@ int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info)
/* send CLC PROPOSAL message across internal TCP socket */
int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,
struct smc_ib_device *ibdev, u8 ibport,
struct smc_ib_device *ibdev, u8 ibport, u8 gid[],
struct smcd_dev *ismdev)
{
struct smc_clc_ipv6_prefix ipv6_prfx[SMC_CLC_MAX_V6_PREFIX];
@ -409,7 +409,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,
/* add SMC-R specifics */
memcpy(pclc.lcl.id_for_peer, local_systemid,
sizeof(local_systemid));
memcpy(&pclc.lcl.gid, &ibdev->gid[ibport - 1], SMC_GID_SIZE);
memcpy(&pclc.lcl.gid, gid, SMC_GID_SIZE);
memcpy(&pclc.lcl.mac, &ibdev->mac[ibport - 1], ETH_ALEN);
pclc.iparea_offset = htons(0);
}
@ -492,8 +492,7 @@ int smc_clc_send_confirm(struct smc_sock *smc)
cclc.hdr.length = htons(SMCR_CLC_ACCEPT_CONFIRM_LEN);
memcpy(cclc.lcl.id_for_peer, local_systemid,
sizeof(local_systemid));
memcpy(&cclc.lcl.gid, &link->smcibdev->gid[link->ibport - 1],
SMC_GID_SIZE);
memcpy(&cclc.lcl.gid, link->gid, SMC_GID_SIZE);
memcpy(&cclc.lcl.mac, &link->smcibdev->mac[link->ibport - 1],
ETH_ALEN);
hton24(cclc.qpn, link->roce_qp->qp_num);
@ -566,8 +565,7 @@ int smc_clc_send_accept(struct smc_sock *new_smc, int srv_first_contact)
link = &conn->lgr->lnk[SMC_SINGLE_LINK];
memcpy(aclc.lcl.id_for_peer, local_systemid,
sizeof(local_systemid));
memcpy(&aclc.lcl.gid, &link->smcibdev->gid[link->ibport - 1],
SMC_GID_SIZE);
memcpy(&aclc.lcl.gid, link->gid, SMC_GID_SIZE);
memcpy(&aclc.lcl.mac, link->smcibdev->mac[link->ibport - 1],
ETH_ALEN);
hton24(aclc.qpn, link->roce_qp->qp_num);

2
net/smc/smc_clc.h
View File

@ -179,7 +179,7 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen,
u8 expected_type);
int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info);
int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,
struct smc_ib_device *smcibdev, u8 ibport,
struct smc_ib_device *smcibdev, u8 ibport, u8 gid[],
struct smcd_dev *ismdev);
int smc_clc_send_confirm(struct smc_sock *smc);
int smc_clc_send_accept(struct smc_sock *smc, int srv_first_contact);

37
net/smc/smc_core.c
View File

@ -219,6 +219,10 @@ static int smc_lgr_create(struct smc_sock *smc, bool is_smcd,
get_random_bytes(rndvec, sizeof(rndvec));
lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +
(rndvec[2] << 16);
rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,
vlan_id, lnk->gid, &lnk->sgid_index);
if (rc)
goto free_lgr;
rc = smc_llc_link_init(lnk);
if (rc)
goto free_lgr;
@ -522,37 +526,6 @@ out:
return rc;
}
/* determine the link gid matching the vlan id of the link group */
static int smc_link_determine_gid(struct smc_link_group *lgr)
{
struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
struct ib_gid_attr gattr;
union ib_gid gid;
int i;
if (!lgr->vlan_id) {
lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1];
return 0;
}
for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len;
i++) {
if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid,
&gattr))
continue;
if (gattr.ndev) {
if (is_vlan_dev(gattr.ndev) &&
vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id) {
lnk->gid = gid;
dev_put(gattr.ndev);
return 0;
}
dev_put(gattr.ndev);
}
}
return -ENODEV;
}
static bool smcr_lgr_match(struct smc_link_group *lgr,
struct smc_clc_msg_local *lcl,
enum smc_lgr_role role)
@ -631,8 +604,6 @@ create:
if (rc)
goto out;
smc_lgr_register_conn(conn); /* add smc conn to lgr */
if (!is_smcd)
rc = smc_link_determine_gid(conn->lgr);
}
conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;

5
net/smc/smc_core.h
View File

@ -84,14 +84,15 @@ struct smc_link {
wait_queue_head_t wr_reg_wait; /* wait for wr_reg result */
enum smc_wr_reg_state wr_reg_state; /* state of wr_reg request */
union ib_gid gid; /* gid matching used vlan id */
u8 gid[SMC_GID_SIZE];/* gid matching used vlan id*/
u8 sgid_index; /* gid index for vlan id */
u32 peer_qpn; /* QP number of peer */
enum ib_mtu path_mtu; /* used mtu */
enum ib_mtu peer_mtu; /* mtu size of peer */
u32 psn_initial; /* QP tx initial packet seqno */
u32 peer_psn; /* QP rx initial packet seqno */
u8 peer_mac[ETH_ALEN]; /* = gid[8:10||13:15] */
u8 peer_gid[sizeof(union ib_gid)]; /* gid of peer*/
u8 peer_gid[SMC_GID_SIZE]; /* gid of peer*/
u8 link_id; /* unique # within link group */
enum smc_link_state state; /* state of link */

2
net/smc/smc_diag.c
View File

@ -154,7 +154,7 @@ static int __smc_diag_dump(struct sock *sk, struct sk_buff *skb,
smc->conn.lgr->lnk[0].smcibdev->ibdev->name,
sizeof(smc->conn.lgr->lnk[0].smcibdev->ibdev->name));
smc_gid_be16_convert(linfo.lnk[0].gid,
smc->conn.lgr->lnk[0].gid.raw);
smc->conn.lgr->lnk[0].gid);
smc_gid_be16_convert(linfo.lnk[0].peer_gid,
smc->conn.lgr->lnk[0].peer_gid);

41
net/smc/smc_ib.c
View File

@ -68,7 +68,7 @@ static int smc_ib_modify_qp_rtr(struct smc_link *lnk)
qp_attr.path_mtu = min(lnk->path_mtu, lnk->peer_mtu);
qp_attr.ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
rdma_ah_set_port_num(&qp_attr.ah_attr, lnk->ibport);
rdma_ah_set_grh(&qp_attr.ah_attr, NULL, 0, 0, 1, 0);
rdma_ah_set_grh(&qp_attr.ah_attr, NULL, 0, lnk->sgid_index, 1, 0);
rdma_ah_set_dgid_raw(&qp_attr.ah_attr, lnk->peer_gid);
memcpy(&qp_attr.ah_attr.roce.dmac, lnk->peer_mac,
sizeof(lnk->peer_mac));
@ -142,13 +142,13 @@ out:
return rc;
}
static int smc_ib_fill_gid_and_mac(struct smc_ib_device *smcibdev, u8 ibport)
static int smc_ib_fill_mac(struct smc_ib_device *smcibdev, u8 ibport)
{
struct ib_gid_attr gattr;
union ib_gid gid;
int rc;
rc = ib_query_gid(smcibdev->ibdev, ibport, 0,
&smcibdev->gid[ibport - 1], &gattr);
rc = ib_query_gid(smcibdev->ibdev, ibport, 0, &gid, &gattr);
if (rc || !gattr.ndev)
return -ENODEV;
@ -175,6 +175,37 @@ bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)
return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;
}
/* determine the gid for an ib-device port and vlan id */
int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport,
unsigned short vlan_id, u8 gid[], u8 *sgid_index)
{
struct ib_gid_attr gattr;
union ib_gid _gid;
int i;
for (i = 0; i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) {
memset(&_gid, 0, SMC_GID_SIZE);
memset(&gattr, 0, sizeof(gattr));
if (ib_query_gid(smcibdev->ibdev, ibport, i, &_gid, &gattr))
continue;
if (!gattr.ndev)
continue;
if (((!vlan_id && !is_vlan_dev(gattr.ndev)) ||
(vlan_id && is_vlan_dev(gattr.ndev) &&
vlan_dev_vlan_id(gattr.ndev) == vlan_id)) &&
gattr.gid_type == IB_GID_TYPE_IB) {
if (gid)
memcpy(gid, &_gid, SMC_GID_SIZE);
if (sgid_index)
*sgid_index = i;
dev_put(gattr.ndev);
return 0;
}
dev_put(gattr.ndev);
}
return -ENODEV;
}
static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
{
int rc;
@ -186,7 +217,7 @@ static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
if (rc)
goto out;
/* the SMC protocol requires specification of the RoCE MAC address */
rc = smc_ib_fill_gid_and_mac(smcibdev, ibport);
rc = smc_ib_fill_mac(smcibdev, ibport);
if (rc)
goto out;
if (!strncmp(local_systemid, SMC_LOCAL_SYSTEMID_RESET,

3
net/smc/smc_ib.h
View File

@ -40,7 +40,6 @@ struct smc_ib_device { /* ib-device infos for smc */
struct tasklet_struct recv_tasklet; /* called by recv cq handler */
char mac[SMC_MAX_PORTS][ETH_ALEN];
/* mac address per port*/
union ib_gid gid[SMC_MAX_PORTS]; /* gid per port */
u8 pnetid[SMC_MAX_PORTS][SMC_MAX_PNETID_LEN];
/* pnetid per port */
u8 initialized : 1; /* ib dev CQ, evthdl done */
@ -77,4 +76,6 @@ void smc_ib_sync_sg_for_cpu(struct smc_ib_device *smcibdev,
void smc_ib_sync_sg_for_device(struct smc_ib_device *smcibdev,
struct smc_buf_desc *buf_slot,
enum dma_data_direction data_direction);
int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport,
unsigned short vlan_id, u8 gid[], u8 *sgid_index);
#endif

15
net/smc/smc_llc.c
View File

@ -203,8 +203,7 @@ int smc_llc_send_confirm_link(struct smc_link *link,
confllc->hd.flags |= SMC_LLC_FLAG_RESP;
memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
ETH_ALEN);
memcpy(confllc->sender_gid, &link->smcibdev->gid[link->ibport - 1],
SMC_GID_SIZE);
memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
confllc->link_num = link->link_id;
memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
@ -241,8 +240,7 @@ static int smc_llc_send_confirm_rkey(struct smc_link *link,
/* prepare an add link message */
static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
struct smc_link *link, u8 mac[],
union ib_gid *gid,
struct smc_link *link, u8 mac[], u8 gid[],
enum smc_llc_reqresp reqresp)
{
memset(addllc, 0, sizeof(*addllc));
@ -259,8 +257,7 @@ static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
}
/* send ADD LINK request or response */
int smc_llc_send_add_link(struct smc_link *link, u8 mac[],
union ib_gid *gid,
int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
enum smc_llc_reqresp reqresp)
{
struct smc_llc_msg_add_link *addllc;
@ -423,14 +420,12 @@ static void smc_llc_rx_add_link(struct smc_link *link,
if (lgr->role == SMC_SERV) {
smc_llc_prep_add_link(llc, link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_REQ);
link->gid, SMC_LLC_REQ);
} else {
smc_llc_prep_add_link(llc, link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_RESP);
link->gid, SMC_LLC_RESP);
}
smc_llc_send_message(link, llc, sizeof(*llc));
}

2
net/smc/smc_llc.h
View File

@ -38,7 +38,7 @@ enum smc_llc_msg_type {
/* transmit */
int smc_llc_send_confirm_link(struct smc_link *lnk,
enum smc_llc_reqresp reqresp);
int smc_llc_send_add_link(struct smc_link *link, u8 mac[], union ib_gid *gid,
int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
enum smc_llc_reqresp reqresp);
int smc_llc_send_delete_link(struct smc_link *link,
enum smc_llc_reqresp reqresp);

30
net/smc/smc_pnet.c
View File

@ -535,11 +535,13 @@ static struct net_device *pnet_find_base_ndev(struct net_device *ndev)
}
/* Determine the corresponding IB device port based on the hardware PNETID.
* Searching stops at the first matching active IB device port.
* Searching stops at the first matching active IB device port with vlan_id
* configured.
*/
static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev,
struct smc_ib_device **smcibdev,
u8 *ibport)
u8 *ibport, unsigned short vlan_id,
u8 gid[])
{
u8 ndev_pnetid[SMC_MAX_PNETID_LEN];
struct smc_ib_device *ibdev;
@ -553,15 +555,20 @@ static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev,
spin_lock(&smc_ib_devices.lock);
list_for_each_entry(ibdev, &smc_ib_devices.list, list) {
for (i = 1; i <= SMC_MAX_PORTS; i++) {
if (!rdma_is_port_valid(ibdev->ibdev, i))
continue;
if (!memcmp(ibdev->pnetid[i - 1], ndev_pnetid,
SMC_MAX_PNETID_LEN) &&
smc_ib_port_active(ibdev, i)) {
smc_ib_port_active(ibdev, i) &&
!smc_ib_determine_gid(ibdev, i, vlan_id, gid,
NULL)) {
*smcibdev = ibdev;
*ibport = i;
break;
goto out;
}
}
}
out:
spin_unlock(&smc_ib_devices.lock);
}
@ -589,7 +596,8 @@ static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev,
/* Lookup of coupled ib_device via SMC pnet table */
static void smc_pnet_find_roce_by_table(struct net_device *netdev,
struct smc_ib_device **smcibdev,
u8 *ibport)
u8 *ibport, unsigned short vlan_id,
u8 gid[])
{
struct smc_pnetentry *pnetelem;
@ -597,7 +605,10 @@ static void smc_pnet_find_roce_by_table(struct net_device *netdev,
list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) {
if (netdev == pnetelem->ndev) {
if (smc_ib_port_active(pnetelem->smcibdev,
pnetelem->ib_port)) {
pnetelem->ib_port) &&
!smc_ib_determine_gid(pnetelem->smcibdev,
pnetelem->ib_port, vlan_id,
gid, NULL)) {
*smcibdev = pnetelem->smcibdev;
*ibport = pnetelem->ib_port;
}
@ -612,7 +623,8 @@ static void smc_pnet_find_roce_by_table(struct net_device *netdev,
* ethernet interface.
*/
void smc_pnet_find_roce_resource(struct sock *sk,
struct smc_ib_device **smcibdev, u8 *ibport)
struct smc_ib_device **smcibdev, u8 *ibport,
unsigned short vlan_id, u8 gid[])
{
struct dst_entry *dst = sk_dst_get(sk);
@ -625,12 +637,12 @@ void smc_pnet_find_roce_resource(struct sock *sk,
goto out_rel;
/* if possible, lookup via hardware-defined pnetid */
smc_pnet_find_roce_by_pnetid(dst->dev, smcibdev, ibport);
smc_pnet_find_roce_by_pnetid(dst->dev, smcibdev, ibport, vlan_id, gid);
if (*smcibdev)
goto out_rel;
/* lookup via SMC PNET table */
smc_pnet_find_roce_by_table(dst->dev, smcibdev, ibport);
smc_pnet_find_roce_by_table(dst->dev, smcibdev, ibport, vlan_id, gid);
out_rel:
dst_release(dst);

3
net/smc/smc_pnet.h
View File

@ -33,7 +33,8 @@ int smc_pnet_init(void) __init;
void smc_pnet_exit(void);
int smc_pnet_remove_by_ibdev(struct smc_ib_device *ibdev);
void smc_pnet_find_roce_resource(struct sock *sk,
struct smc_ib_device **smcibdev, u8 *ibport);
struct smc_ib_device **smcibdev, u8 *ibport,
unsigned short vlan_id, u8 gid[]);
void smc_pnet_find_ism_resource(struct sock *sk, struct smcd_dev **smcismdev);
#endif