leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'ioat' into dmaengine

Browse Source
This commit is contained in:
Dan Williams 2009-09-08 17:52:57 -07:00
commit e12c4fa377
14 changed files with 2608 additions and 2044 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/dma/Makefile
View File

@ -1,8 +1,7 @@
obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
obj-$(CONFIG_NET_DMA) += iovlock.o
obj-$(CONFIG_DMATEST) += dmatest.o
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
obj-$(CONFIG_INTEL_IOATDMA) += ioat/
obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
obj-$(CONFIG_FSL_DMA) += fsldma.o
obj-$(CONFIG_MV_XOR) += mv_xor.o

2
drivers/dma/ioat/Makefile Normal file
View File

@ -0,0 +1,2 @@
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
ioatdma-objs := pci.o dma.o dma_v2.o dca.o

13
drivers/dma/ioat_dca.c → drivers/dma/ioat/dca.c
View File

@ -33,8 +33,8 @@
#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
#endif
#include "ioatdma.h"
#include "ioatdma_registers.h"
#include "dma.h"
#include "registers.h"
/*
* Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
@ -242,7 +242,8 @@ static struct dca_ops ioat_dca_ops = {
};
struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
struct dca_provider * __devinit
ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
@ -407,7 +408,8 @@ static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
return slots;
}
struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
struct dca_provider * __devinit
ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
@ -602,7 +604,8 @@ static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
return slots;
}
struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
struct dca_provider * __devinit
ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;

1139
drivers/dma/ioat/dma.c Normal file
View File

File diff suppressed because it is too large Load Diff

306
drivers/dma/ioat/dma.h Normal file
View File

@ -0,0 +1,306 @@
/*
* Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* The full GNU General Public License is included in this distribution in the
* file called COPYING.
*/
#ifndef IOATDMA_H
#define IOATDMA_H
#include <linux/dmaengine.h>
#include "hw.h"
#include "registers.h"
#include <linux/init.h>
#include <linux/dmapool.h>
#include <linux/cache.h>
#include <linux/pci_ids.h>
#include <net/tcp.h>
#define IOAT_DMA_VERSION "3.64"
#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
#define IOAT_DMA_DCA_ANY_CPU ~0
#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd)
#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev)
#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
/*
* workaround for IOAT ver.3.0 null descriptor issue
* (channel returns error when size is 0)
*/
#define NULL_DESC_BUFFER_SIZE 1
/**
* struct ioatdma_device - internal representation of a IOAT device
* @pdev: PCI-Express device
* @reg_base: MMIO register space base address
* @dma_pool: for allocating DMA descriptors
* @common: embedded struct dma_device
* @version: version of ioatdma device
* @msix_entries: irq handlers
* @idx: per channel data
* @dca: direct cache access context
* @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
* @enumerate_channels: hw version specific channel enumeration
*/
struct ioatdma_device {
struct pci_dev *pdev;
void __iomem *reg_base;
struct pci_pool *dma_pool;
struct pci_pool *completion_pool;
struct dma_device common;
u8 version;
struct msix_entry msix_entries[4];
struct ioat_chan_common *idx[4];
struct dca_provider *dca;
void (*intr_quirk)(struct ioatdma_device *device);
int (*enumerate_channels)(struct ioatdma_device *device);
};
struct ioat_chan_common {
struct dma_chan common;
void __iomem *reg_base;
unsigned long last_completion;
spinlock_t cleanup_lock;
dma_cookie_t completed_cookie;
unsigned long state;
#define IOAT_COMPLETION_PENDING 0
#define IOAT_COMPLETION_ACK 1
#define IOAT_RESET_PENDING 2
struct timer_list timer;
#define COMPLETION_TIMEOUT msecs_to_jiffies(100)
#define IDLE_TIMEOUT msecs_to_jiffies(2000)
#define RESET_DELAY msecs_to_jiffies(100)
struct ioatdma_device *device;
dma_addr_t completion_dma;
u64 *completion;
struct tasklet_struct cleanup_task;
};
/**
* struct ioat_dma_chan - internal representation of a DMA channel
*/
struct ioat_dma_chan {
struct ioat_chan_common base;
size_t xfercap; /* XFERCAP register value expanded out */
spinlock_t desc_lock;
struct list_head free_desc;
struct list_head used_desc;
int pending;
u16 desccount;
};
static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
{
return container_of(c, struct ioat_chan_common, common);
}
static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
{
struct ioat_chan_common *chan = to_chan_common(c);
return container_of(chan, struct ioat_dma_chan, base);
}
/**
* ioat_is_complete - poll the status of an ioat transaction
* @c: channel handle
* @cookie: transaction identifier
* @done: if set, updated with last completed transaction
* @used: if set, updated with last used transaction
*/
static inline enum dma_status
ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
dma_cookie_t *done, dma_cookie_t *used)
{
struct ioat_chan_common *chan = to_chan_common(c);
dma_cookie_t last_used;
dma_cookie_t last_complete;
last_used = c->cookie;
last_complete = chan->completed_cookie;
if (done)
*done = last_complete;
if (used)
*used = last_used;
return dma_async_is_complete(cookie, last_complete, last_used);
}
/* wrapper around hardware descriptor format + additional software fields */
/**
* struct ioat_desc_sw - wrapper around hardware descriptor
* @hw: hardware DMA descriptor
* @node: this descriptor will either be on the free list,
* or attached to a transaction list (async_tx.tx_list)
* @txd: the generic software descriptor for all engines
* @id: identifier for debug
*/
struct ioat_desc_sw {
struct ioat_dma_descriptor *hw;
struct list_head node;
size_t len;
struct dma_async_tx_descriptor txd;
#ifdef DEBUG
int id;
#endif
};
#ifdef DEBUG
#define set_desc_id(desc, i) ((desc)->id = (i))
#define desc_id(desc) ((desc)->id)
#else
#define set_desc_id(desc, i)
#define desc_id(desc) (0)
#endif
static inline void
__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
struct dma_async_tx_descriptor *tx, int id)
{
struct device *dev = to_dev(chan);
dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
" ctl: %#x (op: %d int_en: %d compl: %d)\n", id,
(unsigned long long) tx->phys,
(unsigned long long) hw->next, tx->cookie, tx->flags,
hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
}
#define dump_desc_dbg(c, d) \
({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; })
static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
{
#ifdef CONFIG_NET_DMA
sysctl_tcp_dma_copybreak = copybreak;
#endif
}
static inline struct ioat_chan_common *
ioat_chan_by_index(struct ioatdma_device *device, int index)
{
return device->idx[index];
}
static inline u64 ioat_chansts(struct ioat_chan_common *chan)
{
u8 ver = chan->device->version;
u64 status;
u32 status_lo;
/* We need to read the low address first as this causes the
* chipset to latch the upper bits for the subsequent read
*/
status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
status <<= 32;
status |= status_lo;
return status;
}
static inline void ioat_start(struct ioat_chan_common *chan)
{
u8 ver = chan->device->version;
writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
}
static inline u64 ioat_chansts_to_addr(u64 status)
{
return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
}
static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
{
return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
}
static inline void ioat_suspend(struct ioat_chan_common *chan)
{
u8 ver = chan->device->version;
writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
}
static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
{
struct ioat_chan_common *chan = &ioat->base;
writel(addr & 0x00000000FFFFFFFF,
chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
writel(addr >> 32,
chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
}
static inline bool is_ioat_active(unsigned long status)
{
return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
}
static inline bool is_ioat_idle(unsigned long status)
{
return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
}
static inline bool is_ioat_halted(unsigned long status)
{
return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
}
static inline bool is_ioat_suspended(unsigned long status)
{
return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
}
/* channel was fatally programmed */
static inline bool is_ioat_bug(unsigned long err)
{
return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
IOAT_CHANERR_LENGTH_ERR));
}
int __devinit ioat_probe(struct ioatdma_device *device);
int __devinit ioat_register(struct ioatdma_device *device);
int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
void __devexit ioat_dma_remove(struct ioatdma_device *device);
struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
void __iomem *iobase);
unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx,
void (*timer_fn)(unsigned long),
void (*tasklet)(unsigned long),
unsigned long ioat);
void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
size_t len, struct ioat_dma_descriptor *hw);
bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
unsigned long *phys_complete);
#endif /* IOATDMA_H */

881
drivers/dma/ioat/dma_v2.c Normal file
View File

@ -0,0 +1,881 @@
/*
* Intel I/OAT DMA Linux driver
* Copyright(c) 2004 - 2009 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
*
* The full GNU General Public License is included in this distribution in
* the file called "COPYING".
*
*/
/*
* This driver supports an Intel I/OAT DMA engine (versions >= 2), which
* does asynchronous data movement and checksumming operations.
*/
#include <linux/init.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/i7300_idle.h>
#include "dma.h"
#include "dma_v2.h"
#include "registers.h"
#include "hw.h"
static int ioat_ring_alloc_order = 8;
module_param(ioat_ring_alloc_order, int, 0644);
MODULE_PARM_DESC(ioat_ring_alloc_order,
"ioat2+: allocate 2^n descriptors per channel (default: n=8)");
static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
module_param(ioat_ring_max_alloc_order, int, 0644);
MODULE_PARM_DESC(ioat_ring_max_alloc_order,
"ioat2+: upper limit for dynamic ring resizing (default: n=16)");
static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
{
void * __iomem reg_base = ioat->base.reg_base;
ioat->pending = 0;
ioat->dmacount += ioat2_ring_pending(ioat);;
ioat->issued = ioat->head;
/* make descriptor updates globally visible before notifying channel */
wmb();
writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
dev_dbg(to_dev(&ioat->base),
"%s: head: %#x tail: %#x issued: %#x count: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
}
static void ioat2_issue_pending(struct dma_chan *chan)
{
struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
spin_lock_bh(&ioat->ring_lock);
if (ioat->pending == 1)
__ioat2_issue_pending(ioat);
spin_unlock_bh(&ioat->ring_lock);
}
/**
* ioat2_update_pending - log pending descriptors
* @ioat: ioat2+ channel
*
* set pending to '1' unless pending is already set to '2', pending == 2
* indicates that submission is temporarily blocked due to an in-flight
* reset. If we are already above the ioat_pending_level threshold then
* just issue pending.
*
* called with ring_lock held
*/
static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
{
if (unlikely(ioat->pending == 2))
return;
else if (ioat2_ring_pending(ioat) > ioat_pending_level)
__ioat2_issue_pending(ioat);
else
ioat->pending = 1;
}
static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
{
struct ioat_ring_ent *desc;
struct ioat_dma_descriptor *hw;
int idx;
if (ioat2_ring_space(ioat) < 1) {
dev_err(to_dev(&ioat->base),
"Unable to start null desc - ring full\n");
return;
}
dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued);
idx = ioat2_desc_alloc(ioat, 1);
desc = ioat2_get_ring_ent(ioat, idx);
hw = desc->hw;
hw->ctl = 0;
hw->ctl_f.null = 1;
hw->ctl_f.int_en = 1;
hw->ctl_f.compl_write = 1;
/* set size to non-zero value (channel returns error when size is 0) */
hw->size = NULL_DESC_BUFFER_SIZE;
hw->src_addr = 0;
hw->dst_addr = 0;
async_tx_ack(&desc->txd);
ioat2_set_chainaddr(ioat, desc->txd.phys);
dump_desc_dbg(ioat, desc);
__ioat2_issue_pending(ioat);
}
static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
{
spin_lock_bh(&ioat->ring_lock);
__ioat2_start_null_desc(ioat);
spin_unlock_bh(&ioat->ring_lock);
}
static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
{
struct ioat_chan_common *chan = &ioat->base;
struct dma_async_tx_descriptor *tx;
struct ioat_ring_ent *desc;
bool seen_current = false;
u16 active;
int i;
dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued);
active = ioat2_ring_active(ioat);
for (i = 0; i < active && !seen_current; i++) {
prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
tx = &desc->txd;
dump_desc_dbg(ioat, desc);
if (tx->cookie) {
ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
chan->completed_cookie = tx->cookie;
tx->cookie = 0;
if (tx->callback) {
tx->callback(tx->callback_param);
tx->callback = NULL;
}
}
if (tx->phys == phys_complete)
seen_current = true;
}
ioat->tail += i;
BUG_ON(!seen_current); /* no active descs have written a completion? */
chan->last_completion = phys_complete;
if (ioat->head == ioat->tail) {
dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
__func__);
clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
}
}
/**
* ioat2_cleanup - clean finished descriptors (advance tail pointer)
* @chan: ioat channel to be cleaned up
*/
static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
{
struct ioat_chan_common *chan = &ioat->base;
unsigned long phys_complete;
prefetch(chan->completion);
if (!spin_trylock_bh(&chan->cleanup_lock))
return;
if (!ioat_cleanup_preamble(chan, &phys_complete)) {
spin_unlock_bh(&chan->cleanup_lock);
return;
}
if (!spin_trylock_bh(&ioat->ring_lock)) {
spin_unlock_bh(&chan->cleanup_lock);
return;
}
__cleanup(ioat, phys_complete);
spin_unlock_bh(&ioat->ring_lock);
spin_unlock_bh(&chan->cleanup_lock);
}
static void ioat2_cleanup_tasklet(unsigned long data)
{
struct ioat2_dma_chan *ioat = (void *) data;
ioat2_cleanup(ioat);
writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
}
static void __restart_chan(struct ioat2_dma_chan *ioat)
{
struct ioat_chan_common *chan = &ioat->base;
/* set the tail to be re-issued */
ioat->issued = ioat->tail;
ioat->dmacount = 0;
set_bit(IOAT_COMPLETION_PENDING, &chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
dev_dbg(to_dev(chan),
"%s: head: %#x tail: %#x issued: %#x count: %#x\n",
__func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
if (ioat2_ring_pending(ioat)) {
struct ioat_ring_ent *desc;
desc = ioat2_get_ring_ent(ioat, ioat->tail);
ioat2_set_chainaddr(ioat, desc->txd.phys);
__ioat2_issue_pending(ioat);
} else
__ioat2_start_null_desc(ioat);
}
static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
{
struct ioat_chan_common *chan = &ioat->base;
unsigned long phys_complete;
u32 status;
status = ioat_chansts(chan);
if (is_ioat_active(status) || is_ioat_idle(status))
ioat_suspend(chan);
while (is_ioat_active(status) || is_ioat_idle(status)) {
status = ioat_chansts(chan);
cpu_relax();
}
if (ioat_cleanup_preamble(chan, &phys_complete))
__cleanup(ioat, phys_complete);
__restart_chan(ioat);
}
static bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
static void ioat2_timer_event(unsigned long data)
{
struct ioat2_dma_chan *ioat = (void *) data;
struct ioat_chan_common *chan = &ioat->base;
spin_lock_bh(&chan->cleanup_lock);
if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
unsigned long phys_complete;
u64 status;
spin_lock_bh(&ioat->ring_lock);
status = ioat_chansts(chan);
/* when halted due to errors check for channel
* programming errors before advancing the completion state
*/
if (is_ioat_halted(status)) {
u32 chanerr;
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
BUG_ON(is_ioat_bug(chanerr));
}
/* if we haven't made progress and we have already
* acknowledged a pending completion once, then be more
* forceful with a restart
*/
if (ioat_cleanup_preamble(chan, &phys_complete))
__cleanup(ioat, phys_complete);
else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
ioat2_restart_channel(ioat);
else {
set_bit(IOAT_COMPLETION_ACK, &chan->state);
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
}
spin_unlock_bh(&ioat->ring_lock);
} else {
u16 active;
/* if the ring is idle, empty, and oversized try to step
* down the size
*/
spin_lock_bh(&ioat->ring_lock);
active = ioat2_ring_active(ioat);
if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
reshape_ring(ioat, ioat->alloc_order-1);
spin_unlock_bh(&ioat->ring_lock);
/* keep shrinking until we get back to our minimum
* default size
*/
if (ioat->alloc_order > ioat_get_alloc_order())
mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
}
spin_unlock_bh(&chan->cleanup_lock);
}
/**
* ioat2_enumerate_channels - find and initialize the device's channels
* @device: the device to be enumerated
*/
static int ioat2_enumerate_channels(struct ioatdma_device *device)
{
struct ioat2_dma_chan *ioat;
struct device *dev = &device->pdev->dev;
struct dma_device *dma = &device->common;
u8 xfercap_log;
int i;
INIT_LIST_HEAD(&dma->channels);
dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
dma->chancnt &= 0x1f; /* bits [4:0] valid */
if (dma->chancnt > ARRAY_SIZE(device->idx)) {
dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
dma->chancnt, ARRAY_SIZE(device->idx));
dma->chancnt = ARRAY_SIZE(device->idx);
}
xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
xfercap_log &= 0x1f; /* bits [4:0] valid */
if (xfercap_log == 0)
return 0;
dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
/* FIXME which i/oat version is i7300? */
#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
dma->chancnt--;
#endif
for (i = 0; i < dma->chancnt; i++) {
ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
if (!ioat)
break;
ioat_init_channel(device, &ioat->base, i,
ioat2_timer_event,
ioat2_cleanup_tasklet,
(unsigned long) ioat);
ioat->xfercap_log = xfercap_log;
spin_lock_init(&ioat->ring_lock);
}
dma->chancnt = i;
return i;
}
static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
{
struct dma_chan *c = tx->chan;
struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
struct ioat_chan_common *chan = &ioat->base;
dma_cookie_t cookie = c->cookie;
cookie++;
if (cookie < 0)
cookie = 1;
tx->cookie = cookie;
c->cookie = cookie;
dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
ioat2_update_pending(ioat);
spin_unlock_bh(&ioat->ring_lock);
return cookie;
}
static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
{
struct ioat_dma_descriptor *hw;
struct ioat_ring_ent *desc;
struct ioatdma_device *dma;
dma_addr_t phys;
dma = to_ioatdma_device(chan->device);
hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
if (!hw)
return NULL;
memset(hw, 0, sizeof(*hw));
desc = kzalloc(sizeof(*desc), flags);
if (!desc) {
pci_pool_free(dma->dma_pool, hw, phys);
return NULL;
}
dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = ioat2_tx_submit_unlock;
desc->hw = hw;
desc->txd.phys = phys;
return desc;
}
static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
{
struct ioatdma_device *dma;
dma = to_ioatdma_device(chan->device);
pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
kfree(desc);
}
static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
{
struct ioat_ring_ent **ring;
int descs = 1 << order;
int i;
if (order > ioat_get_max_alloc_order())
return NULL;
/* allocate the array to hold the software ring */
ring = kcalloc(descs, sizeof(*ring), flags);
if (!ring)
return NULL;
for (i = 0; i < descs; i++) {
ring[i] = ioat2_alloc_ring_ent(c, flags);
if (!ring[i]) {
while (i--)
ioat2_free_ring_ent(ring[i], c);
kfree(ring);
return NULL;
}
set_desc_id(ring[i], i);
}
/* link descs */
for (i = 0; i < descs-1; i++) {
struct ioat_ring_ent *next = ring[i+1];
struct ioat_dma_descriptor *hw = ring[i]->hw;
hw->next = next->txd.phys;
}
ring[i]->hw->next = ring[0]->txd.phys;
return ring;
}
/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
* @chan: channel to be initialized
*/
static int ioat2_alloc_chan_resources(struct dma_chan *c)
{
struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
struct ioat_chan_common *chan = &ioat->base;
struct ioat_ring_ent **ring;
u32 chanerr;
int order;
/* have we already been set up? */
if (ioat->ring)
return 1 << ioat->alloc_order;
/* Setup register to interrupt and write completion status on error */
writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
if (chanerr) {
dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
}
/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
chan->completion = pci_pool_alloc(chan->device->completion_pool,
GFP_KERNEL, &chan->completion_dma);
if (!chan->completion)
return -ENOMEM;
memset(chan->completion, 0, sizeof(*chan->completion));
writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
order = ioat_get_alloc_order();
ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
if (!ring)
return -ENOMEM;
spin_lock_bh(&ioat->ring_lock);
ioat->ring = ring;
ioat->head = 0;
ioat->issued = 0;
ioat->tail = 0;
ioat->pending = 0;
ioat->alloc_order = order;
spin_unlock_bh(&ioat->ring_lock);
tasklet_enable(&chan->cleanup_task);
ioat2_start_null_desc(ioat);
return 1 << ioat->alloc_order;
}
static bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
{
/* reshape differs from normal ring allocation in that we want
* to allocate a new software ring while only
* extending/truncating the hardware ring
*/
struct ioat_chan_common *chan = &ioat->base;
struct dma_chan *c = &chan->common;
const u16 curr_size = ioat2_ring_mask(ioat) + 1;
const u16 active = ioat2_ring_active(ioat);
const u16 new_size = 1 << order;
struct ioat_ring_ent **ring;
u16 i;
if (order > ioat_get_max_alloc_order())
return false;
/* double check that we have at least 1 free descriptor */
if (active == curr_size)
return false;
/* when shrinking, verify that we can hold the current active
* set in the new ring
*/
if (active >= new_size)
return false;
/* allocate the array to hold the software ring */
ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
if (!ring)
return false;
/* allocate/trim descriptors as needed */
if (new_size > curr_size) {
/* copy current descriptors to the new ring */
for (i = 0; i < curr_size; i++) {
u16 curr_idx = (ioat->tail+i) & (curr_size-1);
u16 new_idx = (ioat->tail+i) & (new_size-1);
ring[new_idx] = ioat->ring[curr_idx];
set_desc_id(ring[new_idx], new_idx);
}
/* add new descriptors to the ring */
for (i = curr_size; i < new_size; i++) {
u16 new_idx = (ioat->tail+i) & (new_size-1);
ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
if (!ring[new_idx]) {
while (i--) {
u16 new_idx = (ioat->tail+i) & (new_size-1);
ioat2_free_ring_ent(ring[new_idx], c);
}
kfree(ring);
return false;
}
set_desc_id(ring[new_idx], new_idx);
}
/* hw link new descriptors */
for (i = curr_size-1; i < new_size; i++) {
u16 new_idx = (ioat->tail+i) & (new_size-1);
struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
hw->next = next->txd.phys;
}
} else {
struct ioat_dma_descriptor *hw;
struct ioat_ring_ent *next;
/* copy current descriptors to the new ring, dropping the
* removed descriptors
*/
for (i = 0; i < new_size; i++) {
u16 curr_idx = (ioat->tail+i) & (curr_size-1);
u16 new_idx = (ioat->tail+i) & (new_size-1);
ring[new_idx] = ioat->ring[curr_idx];
set_desc_id(ring[new_idx], new_idx);
}
/* free deleted descriptors */
for (i = new_size; i < curr_size; i++) {
struct ioat_ring_ent *ent;
ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
ioat2_free_ring_ent(ent, c);
}
/* fix up hardware ring */
hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
next = ring[(ioat->tail+new_size) & (new_size-1)];
hw->next = next->txd.phys;
}
dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
__func__, new_size);
kfree(ioat->ring);
ioat->ring = ring;
ioat->alloc_order = order;
return true;
}
/**
* ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
* @idx: gets starting descriptor index on successful allocation
* @ioat: ioat2,3 channel (ring) to operate on
* @num_descs: allocation length
*/
static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
{
struct ioat_chan_common *chan = &ioat->base;
spin_lock_bh(&ioat->ring_lock);
/* never allow the last descriptor to be consumed, we need at
* least one free at all times to allow for on-the-fly ring
* resizing.
*/
while (unlikely(ioat2_ring_space(ioat) <= num_descs)) {
if (reshape_ring(ioat, ioat->alloc_order + 1) &&
ioat2_ring_space(ioat) > num_descs)
break;
if (printk_ratelimit())
dev_dbg(to_dev(chan),
"%s: ring full! num_descs: %d (%x:%x:%x)\n",
__func__, num_descs, ioat->head, ioat->tail,
ioat->issued);
spin_unlock_bh(&ioat->ring_lock);
/* progress reclaim in the allocation failure case we
* may be called under bh_disabled so we need to trigger
* the timer event directly
*/
spin_lock_bh(&chan->cleanup_lock);
if (jiffies > chan->timer.expires &&
timer_pending(&chan->timer)) {
mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
spin_unlock_bh(&chan->cleanup_lock);
ioat2_timer_event((unsigned long) ioat);
} else
spin_unlock_bh(&chan->cleanup_lock);
return -ENOMEM;
}
dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
__func__, num_descs, ioat->head, ioat->tail, ioat->issued);
*idx = ioat2_desc_alloc(ioat, num_descs);
return 0; /* with ioat->ring_lock held */
}
static struct dma_async_tx_descriptor *
ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
dma_addr_t dma_src, size_t len, unsigned long flags)
{
struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
struct ioat_dma_descriptor *hw;
struct ioat_ring_ent *desc;
dma_addr_t dst = dma_dest;
dma_addr_t src = dma_src;
size_t total_len = len;
int num_descs;
u16 idx;
int i;
num_descs = ioat2_xferlen_to_descs(ioat, len);
if (likely(num_descs) &&
ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
/* pass */;
else
return NULL;
for (i = 0; i < num_descs; i++) {
size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
desc = ioat2_get_ring_ent(ioat, idx + i);
hw = desc->hw;
hw->size = copy;
hw->ctl = 0;
hw->src_addr = src;
hw->dst_addr = dst;
len -= copy;
dst += copy;
src += copy;
dump_desc_dbg(ioat, desc);
}
desc->txd.flags = flags;
desc->len = total_len;
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1;
dump_desc_dbg(ioat, desc);
/* we leave the channel locked to ensure in order submission */
return &desc->txd;
}
/**
* ioat2_free_chan_resources - release all the descriptors
* @chan: the channel to be cleaned
*/
static void ioat2_free_chan_resources(struct dma_chan *c)
{
struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
struct ioat_chan_common *chan = &ioat->base;
struct ioatdma_device *ioatdma_device = chan->device;
struct ioat_ring_ent *desc;
const u16 total_descs = 1 << ioat->alloc_order;
int descs;
int i;
/* Before freeing channel resources first check
* if they have been previously allocated for this channel.
*/
if (!ioat->ring)
return;
tasklet_disable(&chan->cleanup_task);
del_timer_sync(&chan->timer);
ioat2_cleanup(ioat);
/* Delay 100ms after reset to allow internal DMA logic to quiesce
* before removing DMA descriptor resources.
*/
writeb(IOAT_CHANCMD_RESET,
chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
mdelay(100);
spin_lock_bh(&ioat->ring_lock);
descs = ioat2_ring_space(ioat);
dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
for (i = 0; i < descs; i++) {
desc = ioat2_get_ring_ent(ioat, ioat->head + i);
ioat2_free_ring_ent(desc, c);
}
if (descs < total_descs)
dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
total_descs - descs);
for (i = 0; i < total_descs - descs; i++) {
desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
dump_desc_dbg(ioat, desc);
ioat2_free_ring_ent(desc, c);
}
kfree(ioat->ring);
ioat->ring = NULL;
ioat->alloc_order = 0;
pci_pool_free(ioatdma_device->completion_pool,
chan->completion,
chan->completion_dma);
spin_unlock_bh(&ioat->ring_lock);
chan->last_completion = 0;
chan->completion_dma = 0;
ioat->pending = 0;
ioat->dmacount = 0;
}
static enum dma_status
ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
dma_cookie_t *done, dma_cookie_t *used)
{
struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
return DMA_SUCCESS;
ioat2_cleanup(ioat);
return ioat_is_complete(c, cookie, done, used);
}
int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
{
struct pci_dev *pdev = device->pdev;
struct dma_device *dma;
struct dma_chan *c;
struct ioat_chan_common *chan;
int err;
device->enumerate_channels = ioat2_enumerate_channels;
dma = &device->common;
dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
dma->device_issue_pending = ioat2_issue_pending;
dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
dma->device_free_chan_resources = ioat2_free_chan_resources;
dma->device_is_tx_complete = ioat2_is_complete;
err = ioat_probe(device);
if (err)
return err;
ioat_set_tcp_copy_break(2048);
list_for_each_entry(c, &dma->channels, device_node) {
chan = to_chan_common(c);
writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
chan->reg_base + IOAT_DCACTRL_OFFSET);
}
err = ioat_register(device);
if (err)
return err;
if (dca)
device->dca = ioat2_dca_init(pdev, device->reg_base);
return err;
}
int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
{
struct pci_dev *pdev = device->pdev;
struct dma_device *dma;
struct dma_chan *c;
struct ioat_chan_common *chan;
int err;
u16 dev_id;
device->enumerate_channels = ioat2_enumerate_channels;
dma = &device->common;
dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
dma->device_issue_pending = ioat2_issue_pending;
dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
dma->device_free_chan_resources = ioat2_free_chan_resources;
dma->device_is_tx_complete = ioat2_is_complete;
/* -= IOAT ver.3 workarounds =- */
/* Write CHANERRMSK_INT with 3E07h to mask out the errors
* that can cause stability issues for IOAT ver.3
*/
pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
/* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
* (workaround for spurious config parity error after restart)
*/
pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
err = ioat_probe(device);
if (err)
return err;
ioat_set_tcp_copy_break(262144);
list_for_each_entry(c, &dma->channels, device_node) {
chan = to_chan_common(c);
writel(IOAT_DMA_DCA_ANY_CPU,
chan->reg_base + IOAT_DCACTRL_OFFSET);
}
err = ioat_register(device);
if (err)
return err;
if (dca)
device->dca = ioat3_dca_init(pdev, device->reg_base);
return err;
}

146
drivers/dma/ioat/dma_v2.h Normal file
View File

@ -0,0 +1,146 @@
/*
* Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* The full GNU General Public License is included in this distribution in the
* file called COPYING.
*/
#ifndef IOATDMA_V2_H
#define IOATDMA_V2_H
#include <linux/dmaengine.h>
#include "dma.h"
#include "hw.h"
extern int ioat_pending_level;
/*
* workaround for IOAT ver.3.0 null descriptor issue
* (channel returns error when size is 0)
*/
#define NULL_DESC_BUFFER_SIZE 1
#define IOAT_MAX_ORDER 16
#define ioat_get_alloc_order() \
(min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
#define ioat_get_max_alloc_order() \
(min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
* @base: common ioat channel parameters
* @xfercap_log; log2 of channel max transfer length (for fast division)
* @head: allocated index
* @issued: hardware notification point
* @tail: cleanup index
* @pending: lock free indicator for issued != head
* @dmacount: identical to 'head' except for occasionally resetting to zero
* @alloc_order: log2 of the number of allocated descriptors
* @ring: software ring buffer implementation of hardware ring
* @ring_lock: protects ring attributes
*/
struct ioat2_dma_chan {
struct ioat_chan_common base;
size_t xfercap_log;
u16 head;
u16 issued;
u16 tail;
u16 dmacount;
u16 alloc_order;
int pending;
struct ioat_ring_ent **ring;
spinlock_t ring_lock;
};
static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
{
struct ioat_chan_common *chan = to_chan_common(c);
return container_of(chan, struct ioat2_dma_chan, base);
}
static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat)
{
return (1 << ioat->alloc_order) - 1;
}
/* count of descriptors in flight with the engine */
static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
{
return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat);
}
/* count of descriptors pending submission to hardware */
static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
{
return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat);
}
static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat)
{
u16 num_descs = ioat2_ring_mask(ioat) + 1;
u16 active = ioat2_ring_active(ioat);
BUG_ON(active > num_descs);
return num_descs - active;
}
/* assumes caller already checked space */
static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len)
{
ioat->head += len;
return ioat->head - len;
}
static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
{
u16 num_descs = len >> ioat->xfercap_log;
num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
return num_descs;
}
struct ioat_ring_ent {
struct ioat_dma_descriptor *hw;
struct dma_async_tx_descriptor txd;
size_t len;
#ifdef DEBUG
int id;
#endif
};
static inline struct ioat_ring_ent *
ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
{
return ioat->ring[idx & ioat2_ring_mask(ioat)];
}
static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
{
struct ioat_chan_common *chan = &ioat->base;
writel(addr & 0x00000000FFFFFFFF,
chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
writel(addr >> 32,
chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
}
int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
#endif /* IOATDMA_V2_H */

45
drivers/dma/ioatdma_hw.h → drivers/dma/ioat/hw.h
View File

@ -23,6 +23,7 @@
/* PCI Configuration Space Values */
#define IOAT_PCI_VID 0x8086
#define IOAT_MMIO_BAR 0
/* CB device ID's */
#define IOAT_PCI_DID_5000 0x1A38
@ -39,32 +40,34 @@
struct ioat_dma_descriptor {
uint32_t size;
uint32_t ctl;
union {
uint32_t ctl;
struct {
unsigned int int_en:1;
unsigned int src_snoop_dis:1;
unsigned int dest_snoop_dis:1;
unsigned int compl_write:1;
unsigned int fence:1;
unsigned int null:1;
unsigned int src_brk:1;
unsigned int dest_brk:1;
unsigned int bundle:1;
unsigned int dest_dca:1;
unsigned int hint:1;
unsigned int rsvd2:13;
unsigned int op:8;
} ctl_f;
};
uint64_t src_addr;
uint64_t dst_addr;
uint64_t next;
uint64_t rsv1;
uint64_t rsv2;
uint64_t user1;
/* store some driver data in an unused portion of the descriptor */
union {
uint64_t user1;
uint64_t tx_cnt;
};
uint64_t user2;
};
#define IOAT_DMA_DESCRIPTOR_CTL_INT_GN 0x00000001
#define IOAT_DMA_DESCRIPTOR_CTL_SRC_SN 0x00000002
#define IOAT_DMA_DESCRIPTOR_CTL_DST_SN 0x00000004
#define IOAT_DMA_DESCRIPTOR_CTL_CP_STS 0x00000008
#define IOAT_DMA_DESCRIPTOR_CTL_FRAME 0x00000010
#define IOAT_DMA_DESCRIPTOR_NUL 0x00000020
#define IOAT_DMA_DESCRIPTOR_CTL_SP_BRK 0x00000040
#define IOAT_DMA_DESCRIPTOR_CTL_DP_BRK 0x00000080
#define IOAT_DMA_DESCRIPTOR_CTL_BNDL 0x00000100
#define IOAT_DMA_DESCRIPTOR_CTL_DCA 0x00000200
#define IOAT_DMA_DESCRIPTOR_CTL_BUFHINT 0x00000400
#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_CONTEXT 0xFF000000
#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_DMA 0x00000000
#define IOAT_DMA_DESCRIPTOR_CTL_CONTEXT_DCA 0x00000001
#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_MASK 0xFF000000
#endif

147
drivers/dma/ioat.c → drivers/dma/ioat/pci.c
View File

@ -30,9 +30,10 @@
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/dca.h>
#include "ioatdma.h"
#include "ioatdma_registers.h"
#include "ioatdma_hw.h"
#include "dma.h"
#include "dma_v2.h"
#include "registers.h"
#include "hw.h"
MODULE_VERSION(IOAT_DMA_VERSION);
MODULE_LICENSE("GPL");
@ -60,119 +61,101 @@ static struct pci_device_id ioat_pci_tbl[] = {
{ 0, }
};
struct ioat_device {
struct pci_dev *pdev;
void __iomem *iobase;
struct ioatdma_device *dma;
struct dca_provider *dca;
};
static int __devinit ioat_probe(struct pci_dev *pdev,
const struct pci_device_id *id);
static int __devinit ioat_pci_probe(struct pci_dev *pdev,
const struct pci_device_id *id);
static void __devexit ioat_remove(struct pci_dev *pdev);
static int ioat_dca_enabled = 1;
module_param(ioat_dca_enabled, int, 0644);
MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
#define DRV_NAME "ioatdma"
static struct pci_driver ioat_pci_driver = {
.name = "ioatdma",
.name = DRV_NAME,
.id_table = ioat_pci_tbl,
.probe = ioat_probe,
.probe = ioat_pci_probe,
.remove = __devexit_p(ioat_remove),
};
static int __devinit ioat_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
static struct ioatdma_device *
alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
{
void __iomem *iobase;
struct ioat_device *device;
unsigned long mmio_start, mmio_len;
struct device *dev = &pdev->dev;
struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
if (!d)
return NULL;
d->pdev = pdev;
d->reg_base = iobase;
return d;
}
static int __devinit ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
void __iomem * const *iomap;
struct device *dev = &pdev->dev;
struct ioatdma_device *device;
int err;
err = pci_enable_device(pdev);
err = pcim_enable_device(pdev);
if (err)
goto err_enable_device;
return err;
err = pci_request_regions(pdev, ioat_pci_driver.name);
err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
if (err)
goto err_request_regions;
return err;
iomap = pcim_iomap_table(pdev);
if (!iomap)
return -ENOMEM;
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
if (err)
err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (err)
goto err_set_dma_mask;
return err;
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (err)
err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
if (err)
goto err_set_dma_mask;
return err;
mmio_start = pci_resource_start(pdev, 0);
mmio_len = pci_resource_len(pdev, 0);
iobase = ioremap(mmio_start, mmio_len);
if (!iobase) {
err = -ENOMEM;
goto err_ioremap;
}
device = kzalloc(sizeof(*device), GFP_KERNEL);
if (!device) {
err = -ENOMEM;
goto err_kzalloc;
}
device->pdev = pdev;
pci_set_drvdata(pdev, device);
device->iobase = iobase;
device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL);
if (!device)
return -ENOMEM;
pci_set_master(pdev);
switch (readb(iobase + IOAT_VER_OFFSET)) {
case IOAT_VER_1_2:
device->dma = ioat_dma_probe(pdev, iobase);
if (device->dma && ioat_dca_enabled)
device->dca = ioat_dca_init(pdev, iobase);
break;
case IOAT_VER_2_0:
device->dma = ioat_dma_probe(pdev, iobase);
if (device->dma && ioat_dca_enabled)
device->dca = ioat2_dca_init(pdev, iobase);
break;
case IOAT_VER_3_0:
device->dma = ioat_dma_probe(pdev, iobase);
if (device->dma && ioat_dca_enabled)
device->dca = ioat3_dca_init(pdev, iobase);
break;
default:
err = -ENODEV;
break;
}
if (!device->dma)
err = -ENODEV;
device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
if (!device)
return -ENOMEM;
pci_set_drvdata(pdev, device);
if (err)
goto err_version;
device->version = readb(device->reg_base + IOAT_VER_OFFSET);
if (device->version == IOAT_VER_1_2)
err = ioat1_dma_probe(device, ioat_dca_enabled);
else if (device->version == IOAT_VER_2_0)
err = ioat2_dma_probe(device, ioat_dca_enabled);
else if (device->version >= IOAT_VER_3_0)
err = ioat3_dma_probe(device, ioat_dca_enabled);
else
return -ENODEV;
if (err) {
dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
return -ENODEV;
}
return 0;
err_version:
kfree(device);
err_kzalloc:
iounmap(iobase);
err_ioremap:
err_set_dma_mask:
pci_release_regions(pdev);
pci_disable_device(pdev);
err_request_regions:
err_enable_device:
return err;
}
static void __devexit ioat_remove(struct pci_dev *pdev)
{
struct ioat_device *device = pci_get_drvdata(pdev);
struct ioatdma_device *device = pci_get_drvdata(pdev);
if (!device)
return;
dev_err(&pdev->dev, "Removing dma and dca services\n");
if (device->dca) {
@ -180,13 +163,7 @@ static void __devexit ioat_remove(struct pci_dev *pdev)
free_dca_provider(device->dca);
device->dca = NULL;
}
if (device->dma) {
ioat_dma_remove(device->dma);
device->dma = NULL;
}
kfree(device);
ioat_dma_remove(device);
}
static int __init ioat_init_module(void)

34
drivers/dma/ioatdma_registers.h → drivers/dma/ioat/registers.h
View File

@ -75,7 +75,11 @@
#define IOAT_CHANCTRL_ERR_INT_EN 0x0010
#define IOAT_CHANCTRL_ANY_ERR_ABORT_EN 0x0008
#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004
#define IOAT_CHANCTRL_INT_DISABLE 0x0001
#define IOAT_CHANCTRL_INT_REARM 0x0001
#define IOAT_CHANCTRL_RUN (IOAT_CHANCTRL_INT_REARM |\
IOAT_CHANCTRL_ERR_COMPLETION_EN |\
IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\
IOAT_CHANCTRL_ERR_INT_EN)
#define IOAT_DMA_COMP_OFFSET 0x02 /* 16-bit DMA channel compatibility */
#define IOAT_DMA_COMP_V1 0x0001 /* Compatibility with DMA version 1 */
@ -94,14 +98,14 @@
#define IOAT2_CHANSTS_OFFSET_HIGH 0x0C
#define IOAT_CHANSTS_OFFSET_HIGH(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANSTS_OFFSET_HIGH : IOAT2_CHANSTS_OFFSET_HIGH)
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR ~0x3F
#define IOAT_CHANSTS_SOFT_ERR 0x0000000000000010
#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x0000000000000008
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x0000000000000007
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL)
#define IOAT_CHANSTS_SOFT_ERR 0x10ULL
#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x8ULL
#define IOAT_CHANSTS_STATUS 0x7ULL
#define IOAT_CHANSTS_ACTIVE 0x0
#define IOAT_CHANSTS_DONE 0x1
#define IOAT_CHANSTS_SUSPENDED 0x2
#define IOAT_CHANSTS_HALTED 0x3
@ -204,18 +208,18 @@
#define IOAT_CDAR_OFFSET_HIGH 0x24
#define IOAT_CHANERR_OFFSET 0x28 /* 32-bit Channel Error Register */
#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR 0x0001
#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR 0x0004
#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR 0x0008
#define IOAT_CHANERR_SRC_ADDR_ERR 0x0001
#define IOAT_CHANERR_DEST_ADDR_ERR 0x0002
#define IOAT_CHANERR_NEXT_ADDR_ERR 0x0004
#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR 0x0008
#define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR 0x0010
#define IOAT_CHANERR_CHANCMD_ERR 0x0020
#define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0040
#define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0080
#define IOAT_CHANERR_READ_DATA_ERR 0x0100
#define IOAT_CHANERR_WRITE_DATA_ERR 0x0200
#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR 0x0400
#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR 0x0800
#define IOAT_CHANERR_CONTROL_ERR 0x0400
#define IOAT_CHANERR_LENGTH_ERR 0x0800
#define IOAT_CHANERR_COMPLETION_ADDR_ERR 0x1000
#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
#define IOAT_CHANERR_SOFT_ERR 0x4000

1741
drivers/dma/ioat_dma.c
View File

File diff suppressed because it is too large Load Diff

165
drivers/dma/ioatdma.h
View File

@ -1,165 +0,0 @@
/*
* Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* The full GNU General Public License is included in this distribution in the
* file called COPYING.
*/
#ifndef IOATDMA_H
#define IOATDMA_H
#include <linux/dmaengine.h>
#include "ioatdma_hw.h"
#include <linux/init.h>
#include <linux/dmapool.h>
#include <linux/cache.h>
#include <linux/pci_ids.h>
#include <net/tcp.h>
#define IOAT_DMA_VERSION "3.64"
enum ioat_interrupt {
none = 0,
msix_multi_vector = 1,
msix_single_vector = 2,
msi = 3,
intx = 4,
};
#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
#define IOAT_DMA_DCA_ANY_CPU ~0
#define IOAT_WATCHDOG_PERIOD (2 * HZ)
/**
* struct ioatdma_device - internal representation of a IOAT device
* @pdev: PCI-Express device
* @reg_base: MMIO register space base address
* @dma_pool: for allocating DMA descriptors
* @common: embedded struct dma_device
* @version: version of ioatdma device
* @irq_mode: which style irq to use
* @msix_entries: irq handlers
* @idx: per channel data
*/
struct ioatdma_device {
struct pci_dev *pdev;
void __iomem *reg_base;
struct pci_pool *dma_pool;
struct pci_pool *completion_pool;
struct dma_device common;
u8 version;
enum ioat_interrupt irq_mode;
struct delayed_work work;
struct msix_entry msix_entries[4];
struct ioat_dma_chan *idx[4];
};
/**
* struct ioat_dma_chan - internal representation of a DMA channel
*/
struct ioat_dma_chan {
void __iomem *reg_base;
dma_cookie_t completed_cookie;
unsigned long last_completion;
unsigned long last_completion_time;
size_t xfercap; /* XFERCAP register value expanded out */
spinlock_t cleanup_lock;
spinlock_t desc_lock;
struct list_head free_desc;
struct list_head used_desc;
unsigned long watchdog_completion;
int watchdog_tcp_cookie;
u32 watchdog_last_tcp_cookie;
struct delayed_work work;
int pending;
int dmacount;
int desccount;
struct ioatdma_device *device;
struct dma_chan common;
dma_addr_t completion_addr;
union {
u64 full; /* HW completion writeback */
struct {
u32 low;
u32 high;
};
} *completion_virt;
unsigned long last_compl_desc_addr_hw;
struct tasklet_struct cleanup_task;
};
/* wrapper around hardware descriptor format + additional software fields */
/**
* struct ioat_desc_sw - wrapper around hardware descriptor
* @hw: hardware DMA descriptor
* @node: this descriptor will either be on the free list,
* or attached to a transaction list (async_tx.tx_list)
* @tx_cnt: number of descriptors required to complete the transaction
* @async_tx: the generic software descriptor for all engines
*/
struct ioat_desc_sw {
struct ioat_dma_descriptor *hw;
struct list_head node;
int tx_cnt;
size_t len;
dma_addr_t src;
dma_addr_t dst;
struct dma_async_tx_descriptor async_tx;
};
static inline void ioat_set_tcp_copy_break(struct ioatdma_device *dev)
{
#ifdef CONFIG_NET_DMA
switch (dev->version) {
case IOAT_VER_1_2:
sysctl_tcp_dma_copybreak = 4096;
break;
case IOAT_VER_2_0:
sysctl_tcp_dma_copybreak = 2048;
break;
case IOAT_VER_3_0:
sysctl_tcp_dma_copybreak = 262144;
break;
}
#endif
}
#if defined(CONFIG_INTEL_IOATDMA) || defined(CONFIG_INTEL_IOATDMA_MODULE)
struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
void __iomem *iobase);
void ioat_dma_remove(struct ioatdma_device *device);
struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
#else
#define ioat_dma_probe(pdev, iobase) NULL
#define ioat_dma_remove(device) do { } while (0)
#define ioat_dca_init(pdev, iobase) NULL
#define ioat2_dca_init(pdev, iobase) NULL
#define ioat3_dca_init(pdev, iobase) NULL
#endif
#endif /* IOATDMA_H */

10
drivers/dma/iovlock.c
View File

@ -183,6 +183,11 @@ dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
iov_byte_offset,
kdata,
copy);
/* poll for a descriptor slot */
if (unlikely(dma_cookie < 0)) {
dma_async_issue_pending(chan);
continue;
}
len -= copy;
iov[iovec_idx].iov_len -= copy;
@ -248,6 +253,11 @@ dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
page,
offset,
copy);
/* poll for a descriptor slot */
if (unlikely(dma_cookie < 0)) {
dma_async_issue_pending(chan);
continue;
}
len -= copy;
iov[iovec_idx].iov_len -= copy;

20
drivers/idle/i7300_idle.c
View File

@ -29,8 +29,8 @@
#include <asm/idle.h>
#include "../dma/ioatdma_hw.h"
#include "../dma/ioatdma_registers.h"
#include "../dma/ioat/hw.h"
#include "../dma/ioat/registers.h"
#define I7300_IDLE_DRIVER_VERSION "1.55"
#define I7300_PRINT "i7300_idle:"
@ -126,9 +126,9 @@ static void i7300_idle_ioat_stop(void)
udelay(10);
sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
IOAT_CHANSTS_DMA_TRANSFER_STATUS;
IOAT_CHANSTS_STATUS;
if (sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE)
if (sts != IOAT_CHANSTS_ACTIVE)
break;
}
@ -160,9 +160,9 @@ static int __init i7300_idle_ioat_selftest(u8 *ctl,
udelay(1000);
chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
IOAT_CHANSTS_DMA_TRANSFER_STATUS;
IOAT_CHANSTS_STATUS;
if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE) {
if (chan_sts != IOAT_CHANSTS_DONE) {
/* Not complete, reset the channel */
writeb(IOAT_CHANCMD_RESET,
ioat_chanbase + IOAT1_CHANCMD_OFFSET);
@ -288,9 +288,9 @@ static void __exit i7300_idle_ioat_exit(void)
ioat_chanbase + IOAT1_CHANCMD_OFFSET);
chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
IOAT_CHANSTS_DMA_TRANSFER_STATUS;
IOAT_CHANSTS_STATUS;
if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
if (chan_sts != IOAT_CHANSTS_ACTIVE) {
writew(0, ioat_chanbase + IOAT_CHANCTRL_OFFSET);
break;
}
@ -298,14 +298,14 @@ static void __exit i7300_idle_ioat_exit(void)
}
chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
IOAT_CHANSTS_DMA_TRANSFER_STATUS;
IOAT_CHANSTS_STATUS;
/*
* We tried to reset multiple times. If IO A/T channel is still active
* flag an error and return without cleanup. Memory leak is better
* than random corruption in that extreme error situation.
*/
if (chan_sts == IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
if (chan_sts == IOAT_CHANSTS_ACTIVE) {
printk(KERN_ERR I7300_PRINT "Unable to stop IO A/T channels."
" Not freeing resources\n");
return;