Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-10 22:21:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

RDMA/rxe: Use driver_unregister and new unregistration API

Browse Source 
rxe does not have correct locking for its registration/unregistration
paths, use the core code to handle it instead. In this mode
ib_unregister_device will also do the dealloc, so rxe is required to do
clean up from a callback.

The core code ensures that unregistration is done only once, and generally
takes care of locking and concurrency problems for rxe.

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
This commit is contained in:
Jason Gunthorpe 2019-01-22 16:27:24 -07:00
parent d0899892ed
commit c367074b6c
8 changed files with 30 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
drivers/infiniband/sw/rxe/rxe.c
View File

@ -50,8 +50,10 @@ static void rxe_cleanup_ports(struct rxe_dev *rxe)
/* free resources for a rxe device all objects created for this device must
* have been destroyed
*/
static void rxe_cleanup(struct rxe_dev *rxe)
void rxe_dealloc(struct ib_device *ib_dev)
{
struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);
rxe_pool_cleanup(&rxe->uc_pool);
rxe_pool_cleanup(&rxe->pd_pool);
rxe_pool_cleanup(&rxe->ah_pool);
@ -65,16 +67,10 @@ static void rxe_cleanup(struct rxe_dev *rxe)
rxe_cleanup_ports(rxe);
crypto_free_shash(rxe->tfm);
}
if (rxe->tfm)
crypto_free_shash(rxe->tfm);
/* called when all references have been dropped */
void rxe_release(struct kref *kref)
{
struct rxe_dev *rxe = container_of(kref, struct rxe_dev, ref_cnt);
rxe_cleanup(rxe);
ib_dealloc_device(&rxe->ib_dev);
list_del(&rxe->list);
}
/* initialize rxe device parameters */
@ -312,31 +308,13 @@ int rxe_add(struct rxe_dev *rxe, unsigned int mtu)
{
int err;
kref_init(&rxe->ref_cnt);
err = rxe_init(rxe);
if (err)
goto err1;
return err;
rxe_set_mtu(rxe, mtu);
err = rxe_register_device(rxe);
if (err)
goto err1;
return 0;
err1:
rxe_dev_put(rxe);
return err;
}
/* called by the ifc layer to remove a device */
void rxe_remove(struct rxe_dev *rxe)
{
rxe_unregister_device(rxe);
rxe_dev_put(rxe);
return rxe_register_device(rxe);
}
static int __init rxe_module_init(void)
@ -360,7 +338,7 @@ static int __init rxe_module_init(void)
static void __exit rxe_module_exit(void)
{
rxe_remove_all();
ib_unregister_driver(RDMA_DRIVER_RXE);
rxe_net_exit();
rxe_cache_exit();

6
drivers/infiniband/sw/rxe/rxe.h
View File

@ -96,15 +96,9 @@ static inline u32 rxe_crc32(struct rxe_dev *rxe,
void rxe_set_mtu(struct rxe_dev *rxe, unsigned int dev_mtu);
int rxe_add(struct rxe_dev *rxe, unsigned int mtu);
void rxe_remove(struct rxe_dev *rxe);
void rxe_remove_all(void);
void rxe_rcv(struct sk_buff *skb);
static inline void rxe_dev_put(struct rxe_dev *rxe)
{
kref_put(&rxe->ref_cnt, rxe_release);
}
struct rxe_dev *get_rxe_by_name(const char *name);
/* The caller must do a matching ib_device_put(&dev->ib_dev) */

2
drivers/infiniband/sw/rxe/rxe_loc.h
View File

@ -232,7 +232,7 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
struct ib_srq_attr *attr, enum ib_srq_attr_mask mask,
struct rxe_modify_srq_cmd *ucmd);
void rxe_release(struct kref *kref);
void rxe_dealloc(struct ib_device *ib_dev);
int rxe_completer(void *arg);
int rxe_requester(void *arg);

21
drivers/infiniband/sw/rxe/rxe_net.c
View File

@ -559,21 +559,6 @@ struct rxe_dev *rxe_net_add(struct net_device *ndev)
return rxe;
}
void rxe_remove_all(void)
{
spin_lock_bh(&dev_list_lock);
while (!list_empty(&rxe_dev_list)) {
struct rxe_dev *rxe =
list_first_entry(&rxe_dev_list, struct rxe_dev, list);
list_del(&rxe->list);
spin_unlock_bh(&dev_list_lock);
rxe_remove(rxe);
spin_lock_bh(&dev_list_lock);
}
spin_unlock_bh(&dev_list_lock);
}
static void rxe_port_event(struct rxe_dev *rxe,
enum ib_event_type event)
{
@ -631,10 +616,8 @@ static int rxe_notify(struct notifier_block *not_blk,
switch (event) {
case NETDEV_UNREGISTER:
list_del(&rxe->list);
ib_device_put(&rxe->ib_dev);
rxe_remove(rxe);
return NOTIFY_OK;
ib_unregister_device_queued(&rxe->ib_dev);
break;
case NETDEV_UP:
rxe_port_up(rxe);
break;

24
drivers/infiniband/sw/rxe/rxe_pool.c
View File

@ -398,25 +398,27 @@ void *rxe_alloc(struct rxe_pool *pool)
kref_get(&pool->ref_cnt);
read_unlock_irqrestore(&pool->pool_lock, flags);
kref_get(&pool->rxe->ref_cnt);
if (!ib_device_try_get(&pool->rxe->ib_dev))
goto out_put_pool;
if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
goto out_put_pool;
goto out_cnt;
elem = kmem_cache_zalloc(pool_cache(pool),
(pool->flags & RXE_POOL_ATOMIC) ?
GFP_ATOMIC : GFP_KERNEL);
if (!elem)
goto out_put_pool;
goto out_cnt;
elem->pool = pool;
kref_init(&elem->ref_cnt);
return elem;
out_put_pool:
out_cnt:
atomic_dec(&pool->num_elem);
rxe_dev_put(pool->rxe);
ib_device_put(&pool->rxe->ib_dev);
out_put_pool:
rxe_pool_put(pool);
return NULL;
}
@ -435,19 +437,21 @@ int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
kref_get(&pool->ref_cnt);
read_unlock_irqrestore(&pool->pool_lock, flags);
kref_get(&pool->rxe->ref_cnt);
if (!ib_device_try_get(&pool->rxe->ib_dev))
goto out_put_pool;
if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
goto out_put_pool;
goto out_cnt;
elem->pool = pool;
kref_init(&elem->ref_cnt);
return 0;
out_put_pool:
out_cnt:
atomic_dec(&pool->num_elem);
rxe_dev_put(pool->rxe);
ib_device_put(&pool->rxe->ib_dev);
out_put_pool:
rxe_pool_put(pool);
return -EINVAL;
}
@ -464,7 +468,7 @@ void rxe_elem_release(struct kref *kref)
if (!(pool->flags & RXE_POOL_NO_ALLOC))
kmem_cache_free(pool_cache(pool), elem);
atomic_dec(&pool->num_elem);
rxe_dev_put(pool->rxe);
ib_device_put(&pool->rxe->ib_dev);
rxe_pool_put(pool);
}

5
drivers/infiniband/sw/rxe/rxe_sysfs.c
View File

@ -111,7 +111,7 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
if (strncmp("all", intf, len) == 0) {
pr_info("rxe_sys: remove all");
rxe_remove_all();
ib_unregister_driver(RDMA_DRIVER_RXE);
return 0;
}
@ -122,8 +122,7 @@ static int rxe_param_set_remove(const char *val, const struct kernel_param *kp)
return -EINVAL;
}
list_del(&rxe->list);
rxe_remove(rxe);
ib_unregister_device(&rxe->ib_dev);
return 0;
}

17
drivers/infiniband/sw/rxe/rxe_verbs.c
View File

@ -1135,6 +1135,7 @@ static const struct ib_device_ops rxe_dev_ops = {
.create_cq = rxe_create_cq,
.create_qp = rxe_create_qp,
.create_srq = rxe_create_srq,
.dealloc_driver = rxe_dealloc,
.dealloc_pd = rxe_dealloc_pd,
.dealloc_ucontext = rxe_dealloc_ucontext,
.dereg_mr = rxe_dereg_mr,
@ -1241,22 +1242,8 @@ int rxe_register_device(struct rxe_dev *rxe)
rdma_set_device_sysfs_group(dev, &rxe_attr_group);
dev->driver_id = RDMA_DRIVER_RXE;
err = ib_register_device(dev, "rxe%d");
if (err) {
if (err)
pr_warn("%s failed with error %d\n", __func__, err);
goto err1;
}
return 0;
err1:
crypto_free_shash(rxe->tfm);
return err;
}
void rxe_unregister_device(struct rxe_dev *rxe)
{
struct ib_device *dev = &rxe->ib_dev;
ib_unregister_device(dev);
}

2
drivers/infiniband/sw/rxe/rxe_verbs.h
View File

@ -385,7 +385,6 @@ struct rxe_dev {
struct ib_device_attr attr;
int max_ucontext;
int max_inline_data;
struct kref ref_cnt;
struct mutex usdev_lock;
struct net_device *ndev;
@ -467,7 +466,6 @@ static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
}
int rxe_register_device(struct rxe_dev *rxe);
void rxe_unregister_device(struct rxe_dev *rxe);
void rxe_mc_cleanup(struct rxe_pool_entry *arg);