leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IB/qib: Convert opcode counters to per-context

Browse Source 
This fix changes the opcode relative counters for receive to per
context.

Profiling has shown that when mulitple contexts are being used there
is a lot of cache activity associated with these counters.

The code formerly kept these counters per port, but only provided the
interface to read per HCA.  This patch converts the read of counters
to per HCA and adds the debugfs hooks to be able to read the file as a
sequence of opcodes.

Reviewed-by: Dean Luick <dean.luick@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
This commit is contained in:
Mike Marciniszyn 2013-06-15 17:07:03 -04:00 committed by Roland Dreier
parent 85caafe307
commit ddb8876589
8 changed files with 284 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/infiniband/hw/qib/Makefile
View File

@ -13,3 +13,4 @@ ib_qib-$(CONFIG_PCI_MSI) += qib_iba6120.o
ib_qib-$(CONFIG_X86_64) += qib_wc_x86_64.o
ib_qib-$(CONFIG_PPC64) += qib_wc_ppc64.o
ib_qib-$(CONFIG_DEBUG_FS) += qib_debugfs.o

36
drivers/infiniband/hw/qib/qib.h
View File

@ -115,6 +115,11 @@ struct qib_eep_log_mask {
/*
* Below contains all data related to a single context (formerly called port).
*/
#ifdef CONFIG_DEBUG_FS
struct qib_opcode_stats_perctx;
#endif
struct qib_ctxtdata {
void **rcvegrbuf;
dma_addr_t *rcvegrbuf_phys;
@ -225,12 +230,15 @@ struct qib_ctxtdata {
u8 redirect_seq_cnt;
/* ctxt rcvhdrq head offset */
u32 head;
u32 pkt_count;
/* lookaside fields */
struct qib_qp *lookaside_qp;
u32 lookaside_qpn;
/* QPs waiting for context processing */
struct list_head qp_wait_list;
#ifdef CONFIG_DEBUG_FS
/* verbs stats per CTX */
struct qib_opcode_stats_perctx *opstats;
#endif
};
struct qib_sge_state;
@ -1495,27 +1503,23 @@ extern struct mutex qib_mutex;
* first to avoid possible serial port delays from printk.
*/
#define qib_early_err(dev, fmt, ...) \
do { \
dev_err(dev, fmt, ##__VA_ARGS__); \
} while (0)
dev_err(dev, fmt, ##__VA_ARGS__)
#define qib_dev_err(dd, fmt, ...) \
do { \
dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
qib_get_unit_name((dd)->unit), ##__VA_ARGS__); \
} while (0)
dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
#define qib_dev_warn(dd, fmt, ...) \
dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
#define qib_dev_porterr(dd, port, fmt, ...) \
do { \
dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \
qib_get_unit_name((dd)->unit), (dd)->unit, (port), \
##__VA_ARGS__); \
} while (0)
dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \
qib_get_unit_name((dd)->unit), (dd)->unit, (port), \
##__VA_ARGS__)
#define qib_devinfo(pcidev, fmt, ...) \
do { \
dev_info(&(pcidev)->dev, fmt, ##__VA_ARGS__); \
} while (0)
dev_info(&(pcidev)->dev, fmt, ##__VA_ARGS__)
/*
* this is used for formatting hw error messages...

166
drivers/infiniband/hw/qib/qib_debugfs.c Normal file
View File

@ -0,0 +1,166 @@
#ifdef CONFIG_DEBUG_FS
/*
* Copyright (c) 2013 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include "qib.h"
#include "qib_verbs.h"
#include "qib_debugfs.h"
static struct dentry *qib_dbg_root;
#define DEBUGFS_FILE(name) \
static const struct seq_operations _##name##_seq_ops = { \
.start = _##name##_seq_start, \
.next = _##name##_seq_next, \
.stop = _##name##_seq_stop, \
.show = _##name##_seq_show \
}; \
static int _##name##_open(struct inode *inode, struct file *s) \
{ \
struct seq_file *seq; \
int ret; \
ret = seq_open(s, &_##name##_seq_ops); \
if (ret) \
return ret; \
seq = s->private_data; \
seq->private = inode->i_private; \
return 0; \
} \
static const struct file_operations _##name##_file_ops = { \
.owner = THIS_MODULE, \
.open = _##name##_open, \
.read = seq_read, \
.llseek = seq_lseek, \
.release = seq_release \
};
#define DEBUGFS_FILE_CREATE(name) \
do { \
struct dentry *ent; \
ent = debugfs_create_file(#name , 0400, ibd->qib_ibdev_dbg, \
ibd, &_##name##_file_ops); \
if (!ent) \
pr_warn("create of " #name " failed\n"); \
} while (0)
static void *_opcode_stats_seq_start(struct seq_file *s, loff_t *pos)
{
struct qib_opcode_stats_perctx *opstats;
if (*pos >= ARRAY_SIZE(opstats->stats))
return NULL;
return pos;
}
static void *_opcode_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
{
struct qib_opcode_stats_perctx *opstats;
++*pos;
if (*pos >= ARRAY_SIZE(opstats->stats))
return NULL;
return pos;
}
static void _opcode_stats_seq_stop(struct seq_file *s, void *v)
{
/* nothing allocated */
}
static int _opcode_stats_seq_show(struct seq_file *s, void *v)
{
loff_t *spos = v;
loff_t i = *spos, j;
u64 n_packets = 0, n_bytes = 0;
struct qib_ibdev *ibd = (struct qib_ibdev *)s->private;
struct qib_devdata *dd = dd_from_dev(ibd);
for (j = 0; j < dd->first_user_ctxt; j++) {
if (!dd->rcd[j])
continue;
n_packets += dd->rcd[j]->opstats->stats[i].n_packets;
n_bytes += dd->rcd[j]->opstats->stats[i].n_bytes;
}
if (!n_packets && !n_bytes)
return SEQ_SKIP;
seq_printf(s, "%02llx %llu/%llu\n", i,
(unsigned long long) n_packets,
(unsigned long long) n_bytes);
return 0;
}
DEBUGFS_FILE(opcode_stats)
void qib_dbg_ibdev_init(struct qib_ibdev *ibd)
{
char name[10];
snprintf(name, sizeof(name), "qib%d", dd_from_dev(ibd)->unit);
ibd->qib_ibdev_dbg = debugfs_create_dir(name, qib_dbg_root);
if (!ibd->qib_ibdev_dbg) {
pr_warn("create of %s failed\n", name);
return;
}
DEBUGFS_FILE_CREATE(opcode_stats);
return;
}
void qib_dbg_ibdev_exit(struct qib_ibdev *ibd)
{
if (!qib_dbg_root)
goto out;
debugfs_remove_recursive(ibd->qib_ibdev_dbg);
out:
ibd->qib_ibdev_dbg = NULL;
}
void qib_dbg_init(void)
{
qib_dbg_root = debugfs_create_dir(QIB_DRV_NAME, NULL);
if (!qib_dbg_root)
pr_warn("init of debugfs failed\n");
}
void qib_dbg_exit(void)
{
debugfs_remove_recursive(qib_dbg_root);
qib_dbg_root = NULL;
}
#endif

45
drivers/infiniband/hw/qib/qib_debugfs.h Normal file
View File

@ -0,0 +1,45 @@
#ifndef _QIB_DEBUGFS_H
#define _QIB_DEBUGFS_H
#ifdef CONFIG_DEBUG_FS
/*
* Copyright (c) 2013 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
struct qib_ibdev;
void qib_dbg_ibdev_init(struct qib_ibdev *ibd);
void qib_dbg_ibdev_exit(struct qib_ibdev *ibd);
void qib_dbg_init(void);
void qib_dbg_exit(void);
#endif
#endif /* _QIB_DEBUGFS_H */

1
drivers/infiniband/hw/qib/qib_driver.c
View File

@ -558,7 +558,6 @@ move_along:
}
rcd->head = l;
rcd->pkt_count += i;
/*
* Iterate over all QPs waiting to respond.

41
drivers/infiniband/hw/qib/qib_init.c
View File

@ -46,6 +46,10 @@
#include "qib.h"
#include "qib_common.h"
#include "qib_mad.h"
#ifdef CONFIG_DEBUG_FS
#include "qib_debugfs.h"
#include "qib_verbs.h"
#endif
#undef pr_fmt
#define pr_fmt(fmt) QIB_DRV_NAME ": " fmt
@ -188,7 +192,18 @@ struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *ppd, u32 ctxt,
rcd->cnt = 1;
rcd->ctxt = ctxt;
dd->rcd[ctxt] = rcd;
#ifdef CONFIG_DEBUG_FS
if (ctxt < dd->first_user_ctxt) { /* N/A for PSM contexts */
rcd->opstats = kzalloc_node(sizeof(*rcd->opstats),
GFP_KERNEL, node_id);
if (!rcd->opstats) {
kfree(rcd);
qib_dev_err(dd,
"Unable to allocate per ctxt stats buffer\n");
return NULL;
}
}
#endif
dd->f_init_ctxt(rcd);
/*
@ -959,6 +974,10 @@ void qib_free_ctxtdata(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
vfree(rcd->subctxt_uregbase);
vfree(rcd->subctxt_rcvegrbuf);
vfree(rcd->subctxt_rcvhdr_base);
#ifdef CONFIG_DEBUG_FS
kfree(rcd->opstats);
rcd->opstats = NULL;
#endif
kfree(rcd);
}
@ -1048,7 +1067,6 @@ done:
dd->f_set_armlaunch(dd, 1);
}
void qib_free_devdata(struct qib_devdata *dd)
{
unsigned long flags;
@ -1058,6 +1076,9 @@ void qib_free_devdata(struct qib_devdata *dd)
list_del(&dd->list);
spin_unlock_irqrestore(&qib_devs_lock, flags);
#ifdef CONFIG_DEBUG_FS
qib_dbg_ibdev_exit(&dd->verbs_dev);
#endif
ib_dealloc_device(&dd->verbs_dev.ibdev);
}
@ -1081,6 +1102,10 @@ struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
goto bail;
}
#ifdef CONFIG_DEBUG_FS
qib_dbg_ibdev_init(&dd->verbs_dev);
#endif
idr_preload(GFP_KERNEL);
spin_lock_irqsave(&qib_devs_lock, flags);
@ -1096,6 +1121,9 @@ struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
if (ret < 0) {
qib_early_err(&pdev->dev,
"Could not allocate unit ID: error %d\n", -ret);
#ifdef CONFIG_DEBUG_FS
qib_dbg_ibdev_exit(&dd->verbs_dev);
#endif
ib_dealloc_device(&dd->verbs_dev.ibdev);
dd = ERR_PTR(ret);
goto bail;
@ -1222,6 +1250,9 @@ static int __init qlogic_ib_init(void)
#ifdef CONFIG_INFINIBAND_QIB_DCA
dca_register_notify(&dca_notifier);
#endif
#ifdef CONFIG_DEBUG_FS
qib_dbg_init();
#endif
ret = pci_register_driver(&qib_driver);
if (ret < 0) {
@ -1237,6 +1268,9 @@ static int __init qlogic_ib_init(void)
bail_dev:
#ifdef CONFIG_INFINIBAND_QIB_DCA
dca_unregister_notify(&dca_notifier);
#endif
#ifdef CONFIG_DEBUG_FS
qib_dbg_exit();
#endif
idr_destroy(&qib_unit_table);
qib_dev_cleanup();
@ -1263,6 +1297,9 @@ static void __exit qlogic_ib_cleanup(void)
dca_unregister_notify(&dca_notifier);
#endif
pci_unregister_driver(&qib_driver);
#ifdef CONFIG_DEBUG_FS
qib_dbg_exit();
#endif
qib_cpulist_count = 0;
kfree(qib_cpulist);

8
drivers/infiniband/hw/qib/qib_verbs.c
View File

@ -645,9 +645,11 @@ void qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen)
} else
goto drop;
opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
ibp->opstats[opcode & 0x7f].n_bytes += tlen;
ibp->opstats[opcode & 0x7f].n_packets++;
opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0x7f;
#ifdef CONFIG_DEBUG_FS
rcd->opstats->stats[opcode].n_bytes += tlen;
rcd->opstats->stats[opcode].n_packets++;
#endif
/* Get the destination QP number. */
qp_num = be32_to_cpu(ohdr->bth[1]) & QIB_QPN_MASK;

9
drivers/infiniband/hw/qib/qib_verbs.h
View File

@ -660,6 +660,10 @@ struct qib_opcode_stats {
u64 n_bytes; /* total number of bytes */
};
struct qib_opcode_stats_perctx {
struct qib_opcode_stats stats[128];
};
struct qib_ibport {
struct qib_qp __rcu *qp0;
struct qib_qp __rcu *qp1;
@ -726,7 +730,6 @@ struct qib_ibport {
u8 vl_high_limit;
u8 sl_to_vl[16];
struct qib_opcode_stats opstats[128];
};
@ -770,6 +773,10 @@ struct qib_ibdev {
spinlock_t n_srqs_lock;
u32 n_mcast_grps_allocated; /* number of mcast groups allocated */
spinlock_t n_mcast_grps_lock;
#ifdef CONFIG_DEBUG_FS
/* per HCA debugfs */
struct dentry *qib_ibdev_dbg;
#endif
};
struct qib_verbs_counters {