Commit 5554b359 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: (24 commits)
  I/OAT: I/OAT version 3.0 support
  I/OAT: tcp_dma_copybreak default value dependent on I/OAT version
  I/OAT: Add watchdog/reset functionality to ioatdma
  iop_adma: cleanup iop_chan_xor_slot_count
  iop_adma: document how to calculate the minimum descriptor pool size
  iop_adma: directly reclaim descriptors on allocation failure
  async_tx: make async_tx_test_ack a boolean routine
  async_tx: remove depend_tx from async_tx_sync_epilog
  async_tx: export async_tx_quiesce
  async_tx: fix handling of the "out of descriptor" condition in async_xor
  async_tx: ensure the xor destination buffer remains dma-mapped
  async_tx: list_for_each_entry_rcu() cleanup
  dmaengine: Driver for the Synopsys DesignWare DMA controller
  dmaengine: Add slave DMA interface
  dmaengine: add DMA_COMPL_SKIP_{SRC,DEST}_UNMAP flags to control dma unmap
  dmaengine: Add dma_client parameter to device_alloc_chan_resources
  dmatest: Simple DMA memcpy test client
  dmaengine: DMA engine driver for Marvell XOR engine
  iop-adma: fix platform driver hotplug/coldplug
  dmaengine: track the number of clients using a channel
  ...

Fixed up conflict in drivers/dca/dca-sysfs.c manually
parents 0f6e38a6 7f1b358a
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
*/ */
#include <linux/clk.h> #include <linux/clk.h>
#include <linux/delay.h> #include <linux/delay.h>
#include <linux/dw_dmac.h>
#include <linux/fb.h> #include <linux/fb.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/platform_device.h> #include <linux/platform_device.h>
...@@ -594,6 +595,17 @@ static void __init genclk_init_parent(struct clk *clk) ...@@ -594,6 +595,17 @@ static void __init genclk_init_parent(struct clk *clk)
clk->parent = parent; clk->parent = parent;
} }
static struct dw_dma_platform_data dw_dmac0_data = {
.nr_channels = 3,
};
static struct resource dw_dmac0_resource[] = {
PBMEM(0xff200000),
IRQ(2),
};
DEFINE_DEV_DATA(dw_dmac, 0);
DEV_CLK(hclk, dw_dmac0, hsb, 10);
/* -------------------------------------------------------------------- /* --------------------------------------------------------------------
* System peripherals * System peripherals
* -------------------------------------------------------------------- */ * -------------------------------------------------------------------- */
...@@ -708,17 +720,6 @@ static struct clk pico_clk = { ...@@ -708,17 +720,6 @@ static struct clk pico_clk = {
.users = 1, .users = 1,
}; };
static struct resource dmaca0_resource[] = {
{
.start = 0xff200000,
.end = 0xff20ffff,
.flags = IORESOURCE_MEM,
},
IRQ(2),
};
DEFINE_DEV(dmaca, 0);
DEV_CLK(hclk, dmaca0, hsb, 10);
/* -------------------------------------------------------------------- /* --------------------------------------------------------------------
* HMATRIX * HMATRIX
* -------------------------------------------------------------------- */ * -------------------------------------------------------------------- */
...@@ -831,7 +832,7 @@ void __init at32_add_system_devices(void) ...@@ -831,7 +832,7 @@ void __init at32_add_system_devices(void)
platform_device_register(&at32_eic0_device); platform_device_register(&at32_eic0_device);
platform_device_register(&smc0_device); platform_device_register(&smc0_device);
platform_device_register(&pdc_device); platform_device_register(&pdc_device);
platform_device_register(&dmaca0_device); platform_device_register(&dw_dmac0_device);
platform_device_register(&at32_tcb0_device); platform_device_register(&at32_tcb0_device);
platform_device_register(&at32_tcb1_device); platform_device_register(&at32_tcb1_device);
...@@ -2032,7 +2033,7 @@ struct clk *at32_clock_list[] = { ...@@ -2032,7 +2033,7 @@ struct clk *at32_clock_list[] = {
&smc0_mck, &smc0_mck,
&pdc_hclk, &pdc_hclk,
&pdc_pclk, &pdc_pclk,
&dmaca0_hclk, &dw_dmac0_hclk,
&pico_clk, &pico_clk,
&pio0_mck, &pio0_mck,
&pio1_mck, &pio1_mck,
......
...@@ -73,15 +73,7 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, ...@@ -73,15 +73,7 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
pr_debug("%s: (sync) len: %zu\n", __func__, len); pr_debug("%s: (sync) len: %zu\n", __func__, len);
/* wait for any prerequisite operations */ /* wait for any prerequisite operations */
if (depend_tx) { async_tx_quiesce(&depend_tx);
/* if ack is already set then we cannot be sure
* we are referring to the correct operation
*/
BUG_ON(async_tx_test_ack(depend_tx));
if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
panic("%s: DMA_ERROR waiting for depend_tx\n",
__func__);
}
dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset; dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset;
src_buf = kmap_atomic(src, KM_USER1) + src_offset; src_buf = kmap_atomic(src, KM_USER1) + src_offset;
...@@ -91,7 +83,7 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, ...@@ -91,7 +83,7 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
kunmap_atomic(dest_buf, KM_USER0); kunmap_atomic(dest_buf, KM_USER0);
kunmap_atomic(src_buf, KM_USER1); kunmap_atomic(src_buf, KM_USER1);
async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); async_tx_sync_epilog(cb_fn, cb_param);
} }
return tx; return tx;
......
...@@ -72,19 +72,11 @@ async_memset(struct page *dest, int val, unsigned int offset, ...@@ -72,19 +72,11 @@ async_memset(struct page *dest, int val, unsigned int offset,
dest_buf = (void *) (((char *) page_address(dest)) + offset); dest_buf = (void *) (((char *) page_address(dest)) + offset);
/* wait for any prerequisite operations */ /* wait for any prerequisite operations */
if (depend_tx) { async_tx_quiesce(&depend_tx);
/* if ack is already set then we cannot be sure
* we are referring to the correct operation
*/
BUG_ON(depend_tx->ack);
if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
panic("%s: DMA_ERROR waiting for depend_tx\n",
__func__);
}
memset(dest_buf, val, len); memset(dest_buf, val, len);
async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); async_tx_sync_epilog(cb_fn, cb_param);
} }
return tx; return tx;
......
...@@ -295,7 +295,7 @@ dma_channel_add_remove(struct dma_client *client, ...@@ -295,7 +295,7 @@ dma_channel_add_remove(struct dma_client *client,
case DMA_RESOURCE_REMOVED: case DMA_RESOURCE_REMOVED:
found = 0; found = 0;
spin_lock_irqsave(&async_tx_lock, flags); spin_lock_irqsave(&async_tx_lock, flags);
list_for_each_entry_rcu(ref, &async_tx_master_list, node) list_for_each_entry(ref, &async_tx_master_list, node)
if (ref->chan == chan) { if (ref->chan == chan) {
/* permit backing devices to go away */ /* permit backing devices to go away */
dma_chan_put(ref->chan); dma_chan_put(ref->chan);
...@@ -608,23 +608,34 @@ async_trigger_callback(enum async_tx_flags flags, ...@@ -608,23 +608,34 @@ async_trigger_callback(enum async_tx_flags flags,
pr_debug("%s: (sync)\n", __func__); pr_debug("%s: (sync)\n", __func__);
/* wait for any prerequisite operations */ /* wait for any prerequisite operations */
if (depend_tx) { async_tx_quiesce(&depend_tx);
/* if ack is already set then we cannot be sure
* we are referring to the correct operation
*/
BUG_ON(async_tx_test_ack(depend_tx));
if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
panic("%s: DMA_ERROR waiting for depend_tx\n",
__func__);
}
async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); async_tx_sync_epilog(cb_fn, cb_param);
} }
return tx; return tx;
} }
EXPORT_SYMBOL_GPL(async_trigger_callback); EXPORT_SYMBOL_GPL(async_trigger_callback);
/**
* async_tx_quiesce - ensure tx is complete and freeable upon return
* @tx - transaction to quiesce
*/
void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
{
if (*tx) {
/* if ack is already set then we cannot be sure
* we are referring to the correct operation
*/
BUG_ON(async_tx_test_ack(*tx));
if (dma_wait_for_async_tx(*tx) == DMA_ERROR)
panic("DMA_ERROR waiting for transaction\n");
async_tx_ack(*tx);
*tx = NULL;
}
}
EXPORT_SYMBOL_GPL(async_tx_quiesce);
module_init(async_tx_init); module_init(async_tx_init);
module_exit(async_tx_exit); module_exit(async_tx_exit);
......
...@@ -35,74 +35,121 @@ ...@@ -35,74 +35,121 @@
* when CONFIG_DMA_ENGINE=n * when CONFIG_DMA_ENGINE=n
*/ */
static __always_inline struct dma_async_tx_descriptor * static __always_inline struct dma_async_tx_descriptor *
do_async_xor(struct dma_device *device, do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
struct dma_chan *chan, struct page *dest, struct page **src_list, unsigned int offset, int src_cnt, size_t len,
unsigned int offset, unsigned int src_cnt, size_t len, enum async_tx_flags flags,
enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, struct dma_async_tx_descriptor *depend_tx,
dma_async_tx_callback cb_fn, void *cb_param) dma_async_tx_callback cb_fn, void *cb_param)
{ {
dma_addr_t dma_dest; struct dma_device *dma = chan->device;
dma_addr_t *dma_src = (dma_addr_t *) src_list; dma_addr_t *dma_src = (dma_addr_t *) src_list;
struct dma_async_tx_descriptor *tx; struct dma_async_tx_descriptor *tx = NULL;
int src_off = 0;
int i; int i;
unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; dma_async_tx_callback _cb_fn;
void *_cb_param;
pr_debug("%s: len: %zu\n", __func__, len); enum async_tx_flags async_flags;
enum dma_ctrl_flags dma_flags;
dma_dest = dma_map_page(device->dev, dest, offset, len, int xor_src_cnt;
DMA_FROM_DEVICE); dma_addr_t dma_dest;
dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_FROM_DEVICE);
for (i = 0; i < src_cnt; i++) for (i = 0; i < src_cnt; i++)
dma_src[i] = dma_map_page(device->dev, src_list[i], offset, dma_src[i] = dma_map_page(dma->dev, src_list[i], offset,
len, DMA_TO_DEVICE); len, DMA_TO_DEVICE);
/* Since we have clobbered the src_list we are committed while (src_cnt) {
* to doing this asynchronously. Drivers force forward progress async_flags = flags;
* in case they can not provide a descriptor dma_flags = 0;
*/ xor_src_cnt = min(src_cnt, dma->max_xor);
tx = device->device_prep_dma_xor(chan, dma_dest, dma_src, src_cnt, len, /* if we are submitting additional xors, leave the chain open,
dma_prep_flags); * clear the callback parameters, and leave the destination
if (!tx) { * buffer mapped
if (depend_tx) */
dma_wait_for_async_tx(depend_tx); if (src_cnt > xor_src_cnt) {
async_flags &= ~ASYNC_TX_ACK;
while (!tx) dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;
tx = device->device_prep_dma_xor(chan, dma_dest, _cb_fn = NULL;
dma_src, src_cnt, len, _cb_param = NULL;
dma_prep_flags); } else {
} _cb_fn = cb_fn;
_cb_param = cb_param;
}
if (_cb_fn)
dma_flags |= DMA_PREP_INTERRUPT;
async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); /* Since we have clobbered the src_list we are committed
* to doing this asynchronously. Drivers force forward progress
* in case they can not provide a descriptor
*/
tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off],
xor_src_cnt, len, dma_flags);
if (unlikely(!tx))
async_tx_quiesce(&depend_tx);
/* spin wait for the preceeding transactions to complete */
while (unlikely(!tx)) {
dma_async_issue_pending(chan);
tx = dma->device_prep_dma_xor(chan, dma_dest,
&dma_src[src_off],
xor_src_cnt, len,
dma_flags);
}
async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn,
_cb_param);
depend_tx = tx;
flags |= ASYNC_TX_DEP_ACK;
if (src_cnt > xor_src_cnt) {
/* drop completed sources */
src_cnt -= xor_src_cnt;
src_off += xor_src_cnt;
/* use the intermediate result a source */
dma_src[--src_off] = dma_dest;
src_cnt++;
} else
break;
}
return tx; return tx;
} }
static void static void
do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
unsigned int src_cnt, size_t len, enum async_tx_flags flags, int src_cnt, size_t len, enum async_tx_flags flags,
struct dma_async_tx_descriptor *depend_tx, dma_async_tx_callback cb_fn, void *cb_param)
dma_async_tx_callback cb_fn, void *cb_param)
{ {
void *_dest;
int i; int i;
int xor_src_cnt;
pr_debug("%s: len: %zu\n", __func__, len); int src_off = 0;
void *dest_buf;
void **srcs = (void **) src_list;
/* reuse the 'src_list' array to convert to buffer pointers */ /* reuse the 'src_list' array to convert to buffer pointers */
for (i = 0; i < src_cnt; i++) for (i = 0; i < src_cnt; i++)
src_list[i] = (struct page *) srcs[i] = page_address(src_list[i]) + offset;
(page_address(src_list[i]) + offset);
/* set destination address */ /* set destination address */
_dest = page_address(dest) + offset; dest_buf = page_address(dest) + offset;
if (flags & ASYNC_TX_XOR_ZERO_DST) if (flags & ASYNC_TX_XOR_ZERO_DST)
memset(_dest, 0, len); memset(dest_buf, 0, len);
xor_blocks(src_cnt, len, _dest, while (src_cnt > 0) {
(void **) src_list); /* process up to 'MAX_XOR_BLOCKS' sources */
xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]);
async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); /* drop completed sources */
src_cnt -= xor_src_cnt;
src_off += xor_src_cnt;
}
async_tx_sync_epilog(cb_fn, cb_param);
} }
/** /**
...@@ -132,106 +179,34 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, ...@@ -132,106 +179,34 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset,
struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR, struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR,
&dest, 1, src_list, &dest, 1, src_list,
src_cnt, len); src_cnt, len);
struct dma_device *device = chan ? chan->device : NULL;
struct dma_async_tx_descriptor *tx = NULL;
dma_async_tx_callback _cb_fn;
void *_cb_param;
unsigned long local_flags;
int xor_src_cnt;
int i = 0, src_off = 0;
BUG_ON(src_cnt <= 1); BUG_ON(src_cnt <= 1);
while (src_cnt) { if (chan) {
local_flags = flags; /* run the xor asynchronously */
if (device) { /* run the xor asynchronously */ pr_debug("%s (async): len: %zu\n", __func__, len);
xor_src_cnt = min(src_cnt, device->max_xor);
/* if we are submitting additional xors
* only set the callback on the last transaction
*/
if (src_cnt > xor_src_cnt) {
local_flags &= ~ASYNC_TX_ACK;
_cb_fn = NULL;
_cb_param = NULL;
} else {
_cb_fn = cb_fn;
_cb_param = cb_param;
}
tx = do_async_xor(device, chan, dest,
&src_list[src_off], offset,
xor_src_cnt, len, local_flags,
depend_tx, _cb_fn, _cb_param);
} else { /* run the xor synchronously */
/* in the sync case the dest is an implied source
* (assumes the dest is at the src_off index)
*/
if (flags & ASYNC_TX_XOR_DROP_DST) {
src_cnt--;
src_off++;
}
/* process up to 'MAX_XOR_BLOCKS' sources */
xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
/* if we are submitting additional xors
* only set the callback on the last transaction
*/
if (src_cnt > xor_src_cnt) {
local_flags &= ~ASYNC_TX_ACK;
_cb_fn = NULL;
_cb_param = NULL;
} else {
_cb_fn = cb_fn;
_cb_param = cb_param;
}
/* wait for any prerequisite operations */
if (depend_tx) {
/* if ack is already set then we cannot be sure
* we are referring to the correct operation
*/
BUG_ON(async_tx_test_ack(depend_tx));
if (dma_wait_for_async_tx(depend_tx) ==
DMA_ERROR)
panic("%s: DMA_ERROR waiting for "
"depend_tx\n",
__func__);
}
do_sync_xor(dest, &src_list[src_off], offset,
xor_src_cnt, len, local_flags, depend_tx,
_cb_fn, _cb_param);
}
/* the previous tx is hidden from the client, return do_async_xor(chan, dest, src_list, offset, src_cnt, len,
* so ack it flags, depend_tx, cb_fn, cb_param);
*/ } else {
if (i && depend_tx) /* run the xor synchronously */
async_tx_ack(depend_tx); pr_debug("%s (sync): len: %zu\n", __func__, len);
depend_tx = tx; /* in the sync case the dest is an implied source
* (assumes the dest is the first source)
*/
if (flags & ASYNC_TX_XOR_DROP_DST) {
src_cnt--;
src_list++;
}
if (src_cnt > xor_src_cnt) { /* wait for any prerequisite operations */
/* drop completed sources */ async_tx_quiesce(&depend_tx);
src_cnt -= xor_src_cnt;
src_off += xor_src_cnt;
/* unconditionally preserve the destination */ do_sync_xor(dest, src_list, offset, src_cnt, len,
flags &= ~ASYNC_TX_XOR_ZERO_DST; flags, cb_fn, cb_param);
/* use the intermediate result a source, but remember return NULL;
* it's dropped, because it's implied, in the sync case
*/
src_list[--src_off] = dest;
src_cnt++;
flags |= ASYNC_TX_XOR_DROP_DST;
} else
src_cnt = 0;
i++;
} }
return tx;
} }
EXPORT_SYMBOL_GPL(async_xor); EXPORT_SYMBOL_GPL(async_xor);
...@@ -285,11 +260,11 @@ async_xor_zero_sum(struct page *dest, struct page **src_list, ...@@ -285,11 +260,11 @@ async_xor_zero_sum(struct page *dest, struct page **src_list,
tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt, tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt,
len, result, len, result,
dma_prep_flags); dma_prep_flags);
if (!tx) { if (unlikely(!tx)) {
if (depend_tx) async_tx_quiesce(&depend_tx);
dma_wait_for_async_tx(depend_tx);
while (!tx) while (!tx)
dma_async_issue_pending(chan);
tx = device->device_prep_dma_zero_sum(chan, tx = device->device_prep_dma_zero_sum(chan,
dma_src, src_cnt, len, result, dma_src, src_cnt, len, result,
dma_prep_flags); dma_prep_flags);
...@@ -307,18 +282,11 @@ async_xor_zero_sum(struct page *dest, struct page **src_list, ...@@ -307,18 +282,11 @@ async_xor_zero_sum(struct page *dest, struct page **src_list,
tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags, tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags,
depend_tx, NULL, NULL); depend_tx, NULL, NULL);
if (tx) { async_tx_quiesce(&tx);
if (dma_wait_for_async_tx(tx) == DMA_ERROR)
panic("%s: DMA_ERROR waiting for tx\n",
__func__);
async_tx_ack(tx);
}
*result = page_is_zero(dest, offset, len) ? 0 : 1; *result = page_is_zero(dest, offset, len) ? 0 : 1;
tx = NULL; async_tx_sync_epilog(cb_fn, cb_param);
async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param);
} }
return tx; return tx;
......
...@@ -28,13 +28,29 @@ ...@@ -28,13 +28,29 @@
#include <linux/device.h> #include <linux/device.h>
#include <linux/dca.h> #include <linux/dca.h>
MODULE_LICENSE("GPL"); #define DCA_VERSION "1.4"
/* For now we're assuming a single, global, DCA provider for the system. */ MODULE_VERSION(DCA_VERSION);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Intel Corporation");
static DEFINE_SPINLOCK(dca_lock); static DEFINE_SPINLOCK(dca_lock);
static struct dca_provider *global_dca = NULL; static LIST_HEAD(dca_providers);
static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
{
struct dca_provider *dca, *ret = NULL;
list_for_each_entry(dca, &dca_providers, node) {
if ((!dev) || (dca->ops->dev_managed(dca, dev))) {
ret = dca;
break;
}
}
return ret;
}
/** /**
* dca_add_requester - add a dca client to the list * dca_add_requester - add a dca client to the list
...@@ -42,25 +58,39 @@ static struct dca_provider *global_dca = NULL; ...@@ -42,25 +58,39 @@ static struct dca_provider *global_dca = NULL;
*/ */
int dca_add_requester(struct device *dev) int dca_add_requester(struct device *dev)
{ {
int err, slot; struct dca_provider *dca;
int err, slot = -ENODEV;
if (!global_dca) if (!dev)
return -ENODEV; return -EFAULT;
spin_lock(&dca_lock); spin_lock(&dca_lock);
slot = global_dca->ops->add_requester(global_dca, dev);
spin_unlock(&dca_lock); /* check if the requester has not been added already */
if (slot < 0) dca = dca_find_provider_by_dev(dev);
if (dca) {
spin_unlock(&dca_lock);
return -EEXIST;
}
list_for_each_entry(dca, &dca_providers, node) {
slot = dca->ops->add_requester(dca, dev);
if (slot >= 0)
break;
}
if (slot < 0) {
spin_unlock(&dca_lock);
return slot; return slot;
}
err = dca_sysfs_add_req(global_dca, dev, slot); err = dca_sysfs_add_req(dca, dev, slot);
if (err) { if (err) {
spin_lock(&dca_lock); dca->ops->remove_requester(dca, dev);
global_dca->ops->remove_requester(global_dca, dev);
spin_unlock(&dca_lock); spin_unlock(&dca_lock);
return err; return err;
} }
spin_unlock(&dca_lock);
return 0; return 0;
} }
EXPORT_SYMBOL_GPL(dca_add_requester); EXPORT_SYMBOL_GPL(dca_add_requester);
...@@ -71,30 +101,78 @@ EXPORT_SYMBOL_GPL(dca_add_requester); ...@@ -71,30 +101,78 @@ EXPORT_SYMBOL_GPL(dca_add_requester);
*/ */
int dca_remove_requester(struct device *dev) int dca_remove_requester(struct device *dev)
{ {
struct dca_provider *dca;
int slot; int slot;
if (!global_dca)
return -ENODEV; if (!dev)
return -EFAULT;
spin_lock(&dca_lock); spin_lock(&dca_lock);
slot = global_dca->ops->remove_requester(global_dca, dev); dca = dca_find_provider_by_dev(dev);
spin_unlock(&dca_lock); if (!dca) {
if (slot < 0) spin_unlock(&dca_lock);
return -ENODEV;
}
slot = dca->ops->remove_requester(dca, dev);
if (slot < 0) {
spin_unlock(&dca_lock);
return slot; return slot;
}
dca_sysfs_remove_req(global_dca, slot); dca_sysfs_remove_req(dca, slot);
spin_unlock(&dca_lock);
return 0; return 0;
} }
EXPORT_SYMBOL_GPL(dca_remove_requester); EXPORT_SYMBOL_GPL(dca_remove_requester);
/** /**
* dca_get_tag - return the dca tag for the given cpu * dca_common_get_tag - return the dca tag (serves both new and old api)
* @dev - the device that wants dca service
* @cpu - the cpuid as returned by get_cpu() * @cpu - the cpuid as returned by get_cpu()
*/ */
u8 dca_get_tag(int cpu) u8 dca_common_get_tag(struct device *dev, int cpu)
{ {
if (!global_dca) struct dca_provider *dca;
u8 tag;
spin_lock(&dca_lock);
dca = dca_find_provider_by_dev(dev);
if (!dca) {
spin_unlock(&dca_lock);
return -ENODEV; return -ENODEV;
return global_dca->ops->get_tag(global_dca, cpu); }
tag = dca->ops->get_tag(dca, dev, cpu);
spin_unlock(&dca_lock);
return tag;
}
/**
* dca3_get_tag - return the dca tag to the requester device
* for the given cpu (new api)
* @dev - the device that wants dca service
* @cpu - the cpuid as returned by get_cpu()
*/
u8 dca3_get_tag(struct device *dev, int cpu)
{
if (!dev)
return -EFAULT;
return dca_common_get_tag(dev, cpu);
}
EXPORT_SYMBOL_GPL(dca3_get_tag);
/**
* dca_get_tag - return the dca tag for the given cpu (old api)
* @cpu - the cpuid as returned by get_cpu()
*/
u8 dca_get_tag(int cpu)
{
struct device *dev = NULL;
return dca_common_get_tag(dev, cpu);
} }
EXPORT_SYMBOL_GPL(dca_get_tag); EXPORT_SYMBOL_GPL(dca_get_tag);
...@@ -140,12 +218,10 @@ int register_dca_provider(struct dca_provider *dca, struct device *dev) ...@@ -140,12 +218,10 @@ int register_dca_provider(struct dca_provider *dca, struct device *dev)
{ {
int err; int err;
if (global_dca)
return -EEXIST;
err = dca_sysfs_add_provider(dca, dev); err = dca_sysfs_add_provider(dca, dev);
if (err) if (err)
return err; return err;
global_dca = dca; list_add(&dca->node, &dca_providers);
blocking_notifier_call_chain(&dca_provider_chain, blocking_notifier_call_chain(&dca_provider_chain,
DCA_PROVIDER_ADD, NULL); DCA_PROVIDER_ADD, NULL);
return 0; return 0;
...@@ -158,11 +234,9 @@ EXPORT_SYMBOL_GPL(register_dca_provider); ...@@ -158,11 +234,9 @@ EXPORT_SYMBOL_GPL(register_dca_provider);
*/ */
void unregister_dca_provider(struct dca_provider *dca) void unregister_dca_provider(struct dca_provider *dca)
{ {
if (!global_dca)
return;
blocking_notifier_call_chain(&dca_provider_chain, blocking_notifier_call_chain(&dca_provider_chain,
DCA_PROVIDER_REMOVE, NULL); DCA_PROVIDER_REMOVE, NULL);
global_dca = NULL; list_del(&dca->node);
dca_sysfs_remove_provider(dca); dca_sysfs_remove_provider(dca);
} }
EXPORT_SYMBOL_GPL(unregister_dca_provider); EXPORT_SYMBOL_GPL(unregister_dca_provider);
...@@ -187,6 +261,7 @@ EXPORT_SYMBOL_GPL(dca_unregister_notify); ...@@ -187,6 +261,7 @@ EXPORT_SYMBOL_GPL(dca_unregister_notify);
static int __init dca_init(void) static int __init dca_init(void)
{ {
printk(KERN_ERR "dca service started, version %s\n", DCA_VERSION);
return dca_sysfs_init(); return dca_sysfs_init();
} }
......
...@@ -13,10 +13,11 @@ static spinlock_t dca_idr_lock; ...@@ -13,10 +13,11 @@ static spinlock_t dca_idr_lock;
int dca_sysfs_add_req(struct dca_provider *dca, struct device *dev, int slot) int dca_sysfs_add_req(struct dca_provider *dca, struct device *dev, int slot)
{ {
struct device *cd; struct device *cd;
static int req_count;
cd = device_create_drvdata(dca_class, dca->cd, cd = device_create_drvdata(dca_class, dca->cd,
MKDEV(0, slot + 1), NULL, MKDEV(0, slot + 1), NULL,
"requester%d", slot); "requester%d", req_count++);
if (IS_ERR(cd)) if (IS_ERR(cd))
return PTR_ERR(cd); return PTR_ERR(cd);
return 0; return 0;
......
...@@ -4,13 +4,14 @@ ...@@ -4,13 +4,14 @@
menuconfig DMADEVICES menuconfig DMADEVICES
bool "DMA Engine support" bool "DMA Engine support"
depends on (PCI && X86) || ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX || PPC depends on !HIGHMEM64G && HAS_DMA
depends on !HIGHMEM64G
help help
DMA engines can do asynchronous data transfers without DMA engines can do asynchronous data transfers without
involving the host CPU. Currently, this framework can be involving the host CPU. Currently, this framework can be
used to offload memory copies in the network stack and used to offload memory copies in the network stack and
RAID operations in the MD driver. RAID operations in the MD driver. This menu only presents
DMA Device drivers supported by the configured arch, it may
be empty in some cases.
if DMADEVICES if DMADEVICES
...@@ -37,6 +38,15 @@ config INTEL_IOP_ADMA ...@@ -37,6 +38,15 @@ config INTEL_IOP_ADMA
help help
Enable support for the Intel(R) IOP Series RAID engines. Enable support for the Intel(R) IOP Series RAID engines.
config DW_DMAC
tristate "Synopsys DesignWare AHB DMA support"
depends on AVR32
select DMA_ENGINE
default y if CPU_AT32AP7000
help
Support the Synopsys DesignWare AHB DMA controller. This
can be integrated in chips such as the Atmel AT32ap7000.
config FSL_DMA config FSL_DMA
bool "Freescale MPC85xx/MPC83xx DMA support" bool "Freescale MPC85xx/MPC83xx DMA support"
depends on PPC depends on PPC
...@@ -46,6 +56,14 @@ config FSL_DMA ...@@ -46,6 +56,14 @@ config FSL_DMA
MPC8560/40, MPC8555, MPC8548 and MPC8641 processors. MPC8560/40, MPC8555, MPC8548 and MPC8641 processors.
The MPC8349, MPC8360 is also supported. The MPC8349, MPC8360 is also supported.
config MV_XOR
bool "Marvell XOR engine support"
depends on PLAT_ORION
select ASYNC_CORE
select DMA_ENGINE
---help---
Enable support for the Marvell XOR engine.
config DMA_ENGINE config DMA_ENGINE
bool bool
...@@ -55,10 +73,19 @@ comment "DMA Clients" ...@@ -55,10 +73,19 @@ comment "DMA Clients"
config NET_DMA config NET_DMA
bool "Network: TCP receive copy offload" bool "Network: TCP receive copy offload"
depends on DMA_ENGINE && NET depends on DMA_ENGINE && NET
default (INTEL_IOATDMA || FSL_DMA)
help help
This enables the use of DMA engines in the network stack to This enables the use of DMA engines in the network stack to
offload receive copy-to-user operations, freeing CPU cycles. offload receive copy-to-user operations, freeing CPU cycles.
Since this is the main user of the DMA engine, it should be enabled;
say Y here. Say Y here if you enabled INTEL_IOATDMA or FSL_DMA, otherwise
say N.
config DMATEST
tristate "DMA Test client"
depends on DMA_ENGINE
help
Simple DMA test client. Say N unless you're debugging a
DMA Device driver.
endif endif
obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_NET_DMA) += iovlock.o
obj-$(CONFIG_DMATEST) += dmatest.o
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
obj-$(CONFIG_FSL_DMA) += fsldma.o obj-$(CONFIG_FSL_DMA) += fsldma.o
obj-$(CONFIG_MV_XOR) += mv_xor.o
obj-$(CONFIG_DW_DMAC) += dw_dmac.o
...@@ -169,12 +169,18 @@ static void dma_client_chan_alloc(struct dma_client *client) ...@@ -169,12 +169,18 @@ static void dma_client_chan_alloc(struct dma_client *client)
enum dma_state_client ack; enum dma_state_client ack;
/* Find a channel */ /* Find a channel */
list_for_each_entry(device, &dma_device_list, global_node) list_for_each_entry(device, &dma_device_list, global_node) {
/* Does the client require a specific DMA controller? */
if (client->slave && client->slave->dma_dev
&& client->slave->dma_dev != device->dev)
continue;
list_for_each_entry(chan, &device->channels, device_node) { list_for_each_entry(chan, &device->channels, device_node) {
if (!dma_chan_satisfies_mask(chan, client->cap_mask)) if (!dma_chan_satisfies_mask(chan, client->cap_mask))
continue; continue;
desc = chan->device->device_alloc_chan_resources(chan); desc = chan->device->device_alloc_chan_resources(
chan, client);
if (desc >= 0) { if (desc >= 0) {
ack = client->event_callback(client, ack = client->event_callback(client,
chan, chan,
...@@ -183,12 +189,14 @@ static void dma_client_chan_alloc(struct dma_client *client) ...@@ -183,12 +189,14 @@ static void dma_client_chan_alloc(struct dma_client *client)
/* we are done once this client rejects /* we are done once this client rejects
* an available resource * an available resource
*/ */
if (ack == DMA_ACK) if (ack == DMA_ACK) {
dma_chan_get(chan); dma_chan_get(chan);
else if (ack == DMA_NAK) chan->client_count++;
} else if (ack == DMA_NAK)
return; return;
} }
} }
}
} }
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
...@@ -272,8 +280,10 @@ static void dma_clients_notify_removed(struct dma_chan *chan) ...@@ -272,8 +280,10 @@ static void dma_clients_notify_removed(struct dma_chan *chan)
/* client was holding resources for this channel so /* client was holding resources for this channel so
* free it * free it
*/ */
if (ack == DMA_ACK) if (ack == DMA_ACK) {
dma_chan_put(chan); dma_chan_put(chan);
chan->client_count--;
}
} }
mutex_unlock(&dma_list_mutex); mutex_unlock(&dma_list_mutex);
...@@ -285,6 +295,10 @@ static void dma_clients_notify_removed(struct dma_chan *chan) ...@@ -285,6 +295,10 @@ static void dma_clients_notify_removed(struct dma_chan *chan)
*/ */
void dma_async_client_register(struct dma_client *client) void dma_async_client_register(struct dma_client *client)
{ {
/* validate client data */
BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) &&
!client->slave);
mutex_lock(&dma_list_mutex); mutex_lock(&dma_list_mutex);
list_add_tail(&client->global_node, &dma_client_list); list_add_tail(&client->global_node, &dma_client_list);
mutex_unlock(&dma_list_mutex); mutex_unlock(&dma_list_mutex);
...@@ -313,8 +327,10 @@ void dma_async_client_unregister(struct dma_client *client) ...@@ -313,8 +327,10 @@ void dma_async_client_unregister(struct dma_client *client)
ack = client->event_callback(client, chan, ack = client->event_callback(client, chan,
DMA_RESOURCE_REMOVED); DMA_RESOURCE_REMOVED);
if (ack == DMA_ACK) if (ack == DMA_ACK) {
dma_chan_put(chan); dma_chan_put(chan);
chan->client_count--;
}
} }
list_del(&client->global_node); list_del(&client->global_node);
...@@ -359,6 +375,10 @@ int dma_async_device_register(struct dma_device *device) ...@@ -359,6 +375,10 @@ int dma_async_device_register(struct dma_device *device)
!device->device_prep_dma_memset); !device->device_prep_dma_memset);
BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) && BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
!device->device_prep_dma_interrupt); !device->device_prep_dma_interrupt);
BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) &&
!device->device_prep_slave_sg);
BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) &&
!device->device_terminate_all);
BUG_ON(!device->device_alloc_chan_resources); BUG_ON(!device->device_alloc_chan_resources);
BUG_ON(!device->device_free_chan_resources); BUG_ON(!device->device_free_chan_resources);
...@@ -378,7 +398,7 @@ int dma_async_device_register(struct dma_device *device) ...@@ -378,7 +398,7 @@ int dma_async_device_register(struct dma_device *device)
chan->chan_id = chancnt++; chan->chan_id = chancnt++;
chan->dev.class = &dma_devclass; chan->dev.class = &dma_devclass;
chan->dev.parent = NULL; chan->dev.parent = device->dev;
snprintf(chan->dev.bus_id, BUS_ID_SIZE, "dma%dchan%d", snprintf(chan->dev.bus_id, BUS_ID_SIZE, "dma%dchan%d",
device->dev_id, chan->chan_id); device->dev_id, chan->chan_id);
...@@ -394,6 +414,7 @@ int dma_async_device_register(struct dma_device *device) ...@@ -394,6 +414,7 @@ int dma_async_device_register(struct dma_device *device)
kref_get(&device->refcount); kref_get(&device->refcount);
kref_get(&device->refcount); kref_get(&device->refcount);
kref_init(&chan->refcount); kref_init(&chan->refcount);
chan->client_count = 0;
chan->slow_ref = 0; chan->slow_ref = 0;
INIT_RCU_HEAD(&chan->rcu); INIT_RCU_HEAD(&chan->rcu);
} }
......
This diff is collapsed.
This diff is collapsed.
/*
* Driver for the Synopsys DesignWare AHB DMA Controller
*
* Copyright (C) 2005-2007 Atmel Corporation
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/dw_dmac.h>
#define DW_DMA_MAX_NR_CHANNELS 8
/*
* Redefine this macro to handle differences between 32- and 64-bit
* addressing, big vs. little endian, etc.
*/
#define DW_REG(name) u32 name; u32 __pad_##name
/* Hardware register definitions. */
struct dw_dma_chan_regs {
DW_REG(SAR); /* Source Address Register */
DW_REG(DAR); /* Destination Address Register */
DW_REG(LLP); /* Linked List Pointer */
u32 CTL_LO; /* Control Register Low */
u32 CTL_HI; /* Control Register High */
DW_REG(SSTAT);
DW_REG(DSTAT);
DW_REG(SSTATAR);
DW_REG(DSTATAR);
u32 CFG_LO; /* Configuration Register Low */
u32 CFG_HI; /* Configuration Register High */
DW_REG(SGR);
DW_REG(DSR);
};
struct dw_dma_irq_regs {
DW_REG(XFER);
DW_REG(BLOCK);
DW_REG(SRC_TRAN);
DW_REG(DST_TRAN);
DW_REG(ERROR);
};
struct dw_dma_regs {
/* per-channel registers */
struct dw_dma_chan_regs CHAN[DW_DMA_MAX_NR_CHANNELS];
/* irq handling */
struct dw_dma_irq_regs RAW; /* r */
struct dw_dma_irq_regs STATUS; /* r (raw & mask) */
struct dw_dma_irq_regs MASK; /* rw (set = irq enabled) */
struct dw_dma_irq_regs CLEAR; /* w (ack, affects "raw") */
DW_REG(STATUS_INT); /* r */
/* software handshaking */
DW_REG(REQ_SRC);
DW_REG(REQ_DST);
DW_REG(SGL_REQ_SRC);
DW_REG(SGL_REQ_DST);
DW_REG(LAST_SRC);
DW_REG(LAST_DST);
/* miscellaneous */
DW_REG(CFG);
DW_REG(CH_EN);
DW_REG(ID);
DW_REG(TEST);
/* optional encoded params, 0x3c8..0x3 */
};
/* Bitfields in CTL_LO */
#define DWC_CTLL_INT_EN (1 << 0) /* irqs enabled? */
#define DWC_CTLL_DST_WIDTH(n) ((n)<<1) /* bytes per element */
#define DWC_CTLL_SRC_WIDTH(n) ((n)<<4)
#define DWC_CTLL_DST_INC (0<<7) /* DAR update/not */
#define DWC_CTLL_DST_DEC (1<<7)
#define DWC_CTLL_DST_FIX (2<<7)
#define DWC_CTLL_SRC_INC (0<<7) /* SAR update/not */
#define DWC_CTLL_SRC_DEC (1<<9)
#define DWC_CTLL_SRC_FIX (2<<9)
#define DWC_CTLL_DST_MSIZE(n) ((n)<<11) /* burst, #elements */
#define DWC_CTLL_SRC_MSIZE(n) ((n)<<14)
#define DWC_CTLL_S_GATH_EN (1 << 17) /* src gather, !FIX */
#define DWC_CTLL_D_SCAT_EN (1 << 18) /* dst scatter, !FIX */
#define DWC_CTLL_FC_M2M (0 << 20) /* mem-to-mem */
#define DWC_CTLL_FC_M2P (1 << 20) /* mem-to-periph */
#define DWC_CTLL_FC_P2M (2 << 20) /* periph-to-mem */
#define DWC_CTLL_FC_P2P (3 << 20) /* periph-to-periph */
/* plus 4 transfer types for peripheral-as-flow-controller */
#define DWC_CTLL_DMS(n) ((n)<<23) /* dst master select */
#define DWC_CTLL_SMS(n) ((n)<<25) /* src master select */
#define DWC_CTLL_LLP_D_EN (1 << 27) /* dest block chain */
#define DWC_CTLL_LLP_S_EN (1 << 28) /* src block chain */
/* Bitfields in CTL_HI */
#define DWC_CTLH_DONE 0x00001000
#define DWC_CTLH_BLOCK_TS_MASK 0x00000fff
/* Bitfields in CFG_LO. Platform-configurable bits are in <linux/dw_dmac.h> */
#define DWC_CFGL_CH_SUSP (1 << 8) /* pause xfer */
#define DWC_CFGL_FIFO_EMPTY (1 << 9) /* pause xfer */
#define DWC_CFGL_HS_DST (1 << 10) /* handshake w/dst */
#define DWC_CFGL_HS_SRC (1 << 11) /* handshake w/src */
#define DWC_CFGL_MAX_BURST(x) ((x) << 20)
#define DWC_CFGL_RELOAD_SAR (1 << 30)
#define DWC_CFGL_RELOAD_DAR (1 << 31)
/* Bitfields in CFG_HI. Platform-configurable bits are in <linux/dw_dmac.h> */
#define DWC_CFGH_DS_UPD_EN (1 << 5)
#define DWC_CFGH_SS_UPD_EN (1 << 6)
/* Bitfields in SGR */
#define DWC_SGR_SGI(x) ((x) << 0)
#define DWC_SGR_SGC(x) ((x) << 20)
/* Bitfields in DSR */
#define DWC_DSR_DSI(x) ((x) << 0)
#define DWC_DSR_DSC(x) ((x) << 20)
/* Bitfields in CFG */
#define DW_CFG_DMA_EN (1 << 0)
#define DW_REGLEN 0x400
struct dw_dma_chan {
struct dma_chan chan;
void __iomem *ch_regs;
u8 mask;
spinlock_t lock;
/* these other elements are all protected by lock */
dma_cookie_t completed;
struct list_head active_list;
struct list_head queue;
struct list_head free_list;
struct dw_dma_slave *dws;
unsigned int descs_allocated;
};
static inline struct dw_dma_chan_regs __iomem *
__dwc_regs(struct dw_dma_chan *dwc)
{
return dwc->ch_regs;
}
#define channel_readl(dwc, name) \
__raw_readl(&(__dwc_regs(dwc)->name))
#define channel_writel(dwc, name, val) \
__raw_writel((val), &(__dwc_regs(dwc)->name))
static inline struct dw_dma_chan *to_dw_dma_chan(struct dma_chan *chan)
{
return container_of(chan, struct dw_dma_chan, chan);
}
struct dw_dma {
struct dma_device dma;
void __iomem *regs;
struct tasklet_struct tasklet;
struct clk *clk;
u8 all_chan_mask;
struct dw_dma_chan chan[0];
};
static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw)
{
return dw->regs;
}
#define dma_readl(dw, name) \
__raw_readl(&(__dw_regs(dw)->name))
#define dma_writel(dw, name, val) \
__raw_writel((val), &(__dw_regs(dw)->name))
#define channel_set_bit(dw, reg, mask) \
dma_writel(dw, reg, ((mask) << 8) | (mask))
#define channel_clear_bit(dw, reg, mask) \
dma_writel(dw, reg, ((mask) << 8) | 0)
static inline struct dw_dma *to_dw_dma(struct dma_device *ddev)
{
return container_of(ddev, struct dw_dma, dma);
}
/* LLI == Linked List Item; a.k.a. DMA block descriptor */
struct dw_lli {
/* values that are not changed by hardware */
dma_addr_t sar;
dma_addr_t dar;
dma_addr_t llp; /* chain to next lli */
u32 ctllo;
/* values that may get written back: */
u32 ctlhi;
/* sstat and dstat can snapshot peripheral register state.
* silicon config may discard either or both...
*/
u32 sstat;
u32 dstat;
};
struct dw_desc {
/* FIRST values the hardware uses */
struct dw_lli lli;
/* THEN values for driver housekeeping */
struct list_head desc_node;
struct dma_async_tx_descriptor txd;
size_t len;
};
static inline struct dw_desc *
txd_to_dw_desc(struct dma_async_tx_descriptor *txd)
{
return container_of(txd, struct dw_desc, txd);
}
...@@ -366,7 +366,8 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor( ...@@ -366,7 +366,8 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
* *
* Return - The number of descriptors allocated. * Return - The number of descriptors allocated.
*/ */
static int fsl_dma_alloc_chan_resources(struct dma_chan *chan) static int fsl_dma_alloc_chan_resources(struct dma_chan *chan,
struct dma_client *client)
{ {
struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan);
LIST_HEAD(tmp_list); LIST_HEAD(tmp_list);
...@@ -809,8 +810,7 @@ static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan) ...@@ -809,8 +810,7 @@ static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan)
if (!src) { if (!src) {
dev_err(fsl_chan->dev, dev_err(fsl_chan->dev,
"selftest: Cannot alloc memory for test!\n"); "selftest: Cannot alloc memory for test!\n");
err = -ENOMEM; return -ENOMEM;
goto out;
} }
dest = src + test_size; dest = src + test_size;
...@@ -820,7 +820,7 @@ static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan) ...@@ -820,7 +820,7 @@ static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan)
chan = &fsl_chan->common; chan = &fsl_chan->common;
if (fsl_dma_alloc_chan_resources(chan) < 1) { if (fsl_dma_alloc_chan_resources(chan, NULL) < 1) {
dev_err(fsl_chan->dev, dev_err(fsl_chan->dev,
"selftest: Cannot alloc resources for DMA\n"); "selftest: Cannot alloc resources for DMA\n");
err = -ENODEV; err = -ENODEV;
...@@ -842,13 +842,13 @@ static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan) ...@@ -842,13 +842,13 @@ static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan)
if (fsl_dma_is_complete(chan, cookie, NULL, NULL) != DMA_SUCCESS) { if (fsl_dma_is_complete(chan, cookie, NULL, NULL) != DMA_SUCCESS) {
dev_err(fsl_chan->dev, "selftest: Time out!\n"); dev_err(fsl_chan->dev, "selftest: Time out!\n");
err = -ENODEV; err = -ENODEV;
goto out; goto free_resources;
} }
/* Test free and re-alloc channel resources */ /* Test free and re-alloc channel resources */
fsl_dma_free_chan_resources(chan); fsl_dma_free_chan_resources(chan);
if (fsl_dma_alloc_chan_resources(chan) < 1) { if (fsl_dma_alloc_chan_resources(chan, NULL) < 1) {
dev_err(fsl_chan->dev, dev_err(fsl_chan->dev,
"selftest: Cannot alloc resources for DMA\n"); "selftest: Cannot alloc resources for DMA\n");
err = -ENODEV; err = -ENODEV;
...@@ -927,8 +927,7 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev, ...@@ -927,8 +927,7 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev,
if (!new_fsl_chan) { if (!new_fsl_chan) {
dev_err(&dev->dev, "No free memory for allocating " dev_err(&dev->dev, "No free memory for allocating "
"dma channels!\n"); "dma channels!\n");
err = -ENOMEM; return -ENOMEM;
goto err;
} }
/* get dma channel register base */ /* get dma channel register base */
...@@ -936,7 +935,7 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev, ...@@ -936,7 +935,7 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev,
if (err) { if (err) {
dev_err(&dev->dev, "Can't get %s property 'reg'\n", dev_err(&dev->dev, "Can't get %s property 'reg'\n",
dev->node->full_name); dev->node->full_name);
goto err; goto err_no_reg;
} }
new_fsl_chan->feature = *(u32 *)match->data; new_fsl_chan->feature = *(u32 *)match->data;
...@@ -958,7 +957,7 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev, ...@@ -958,7 +957,7 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev,
dev_err(&dev->dev, "There is no %d channel!\n", dev_err(&dev->dev, "There is no %d channel!\n",
new_fsl_chan->id); new_fsl_chan->id);
err = -EINVAL; err = -EINVAL;
goto err; goto err_no_chan;
} }
fdev->chan[new_fsl_chan->id] = new_fsl_chan; fdev->chan[new_fsl_chan->id] = new_fsl_chan;
tasklet_init(&new_fsl_chan->tasklet, dma_do_tasklet, tasklet_init(&new_fsl_chan->tasklet, dma_do_tasklet,
...@@ -997,23 +996,26 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev, ...@@ -997,23 +996,26 @@ static int __devinit of_fsl_dma_chan_probe(struct of_device *dev,
if (err) { if (err) {
dev_err(&dev->dev, "DMA channel %s request_irq error " dev_err(&dev->dev, "DMA channel %s request_irq error "
"with return %d\n", dev->node->full_name, err); "with return %d\n", dev->node->full_name, err);
goto err; goto err_no_irq;
} }
} }
err = fsl_dma_self_test(new_fsl_chan); err = fsl_dma_self_test(new_fsl_chan);
if (err) if (err)
goto err; goto err_self_test;
dev_info(&dev->dev, "#%d (%s), irq %d\n", new_fsl_chan->id, dev_info(&dev->dev, "#%d (%s), irq %d\n", new_fsl_chan->id,
match->compatible, new_fsl_chan->irq); match->compatible, new_fsl_chan->irq);
return 0; return 0;
err:
dma_halt(new_fsl_chan); err_self_test:
iounmap(new_fsl_chan->reg_base);
free_irq(new_fsl_chan->irq, new_fsl_chan); free_irq(new_fsl_chan->irq, new_fsl_chan);
err_no_irq:
list_del(&new_fsl_chan->common.device_node); list_del(&new_fsl_chan->common.device_node);
err_no_chan:
iounmap(new_fsl_chan->reg_base);
err_no_reg:
kfree(new_fsl_chan); kfree(new_fsl_chan);
return err; return err;
} }
...@@ -1054,8 +1056,7 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev, ...@@ -1054,8 +1056,7 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
fdev = kzalloc(sizeof(struct fsl_dma_device), GFP_KERNEL); fdev = kzalloc(sizeof(struct fsl_dma_device), GFP_KERNEL);
if (!fdev) { if (!fdev) {
dev_err(&dev->dev, "No enough memory for 'priv'\n"); dev_err(&dev->dev, "No enough memory for 'priv'\n");
err = -ENOMEM; return -ENOMEM;
goto err;
} }
fdev->dev = &dev->dev; fdev->dev = &dev->dev;
INIT_LIST_HEAD(&fdev->common.channels); INIT_LIST_HEAD(&fdev->common.channels);
...@@ -1065,7 +1066,7 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev, ...@@ -1065,7 +1066,7 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
if (err) { if (err) {
dev_err(&dev->dev, "Can't get %s property 'reg'\n", dev_err(&dev->dev, "Can't get %s property 'reg'\n",
dev->node->full_name); dev->node->full_name);
goto err; goto err_no_reg;
} }
dev_info(&dev->dev, "Probe the Freescale DMA driver for %s " dev_info(&dev->dev, "Probe the Freescale DMA driver for %s "
...@@ -1103,6 +1104,7 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev, ...@@ -1103,6 +1104,7 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
err: err:
iounmap(fdev->reg_base); iounmap(fdev->reg_base);
err_no_reg:
kfree(fdev); kfree(fdev);
return err; return err;
} }
......
...@@ -47,6 +47,16 @@ static struct pci_device_id ioat_pci_tbl[] = { ...@@ -47,6 +47,16 @@ static struct pci_device_id ioat_pci_tbl[] = {
/* I/OAT v2 platforms */ /* I/OAT v2 platforms */
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
/* I/OAT v3 platforms */
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
{ 0, } { 0, }
}; };
...@@ -83,6 +93,11 @@ static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase) ...@@ -83,6 +93,11 @@ static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase)
if (device->dma && ioat_dca_enabled) if (device->dma && ioat_dca_enabled)
device->dca = ioat2_dca_init(pdev, iobase); device->dca = ioat2_dca_init(pdev, iobase);
break; break;
case IOAT_VER_3_0:
device->dma = ioat_dma_probe(pdev, iobase);
if (device->dma && ioat_dca_enabled)
device->dca = ioat3_dca_init(pdev, iobase);
break;
default: default:
err = -ENODEV; err = -ENODEV;
break; break;
......
...@@ -37,12 +37,18 @@ ...@@ -37,12 +37,18 @@
#include "ioatdma_registers.h" #include "ioatdma_registers.h"
/* /*
* Bit 16 of a tag map entry is the "valid" bit, if it is set then bits 0:15 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
* contain the bit number of the APIC ID to map into the DCA tag. If the valid * contain the bit number of the APIC ID to map into the DCA tag. If the valid
* bit is not set, then the value must be 0 or 1 and defines the bit in the tag. * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
*/ */
#define DCA_TAG_MAP_VALID 0x80 #define DCA_TAG_MAP_VALID 0x80
#define DCA3_TAG_MAP_BIT_TO_INV 0x80
#define DCA3_TAG_MAP_BIT_TO_SEL 0x40
#define DCA3_TAG_MAP_LITERAL_VAL 0x1
#define DCA_TAG_MAP_MASK 0xDF
/* /*
* "Legacy" DCA systems do not implement the DCA register set in the * "Legacy" DCA systems do not implement the DCA register set in the
* I/OAT device. Software needs direct support for their tag mappings. * I/OAT device. Software needs direct support for their tag mappings.
...@@ -95,6 +101,7 @@ struct ioat_dca_slot { ...@@ -95,6 +101,7 @@ struct ioat_dca_slot {
}; };
#define IOAT_DCA_MAX_REQ 6 #define IOAT_DCA_MAX_REQ 6
#define IOAT3_DCA_MAX_REQ 2
struct ioat_dca_priv { struct ioat_dca_priv {
void __iomem *iobase; void __iomem *iobase;
...@@ -171,7 +178,9 @@ static int ioat_dca_remove_requester(struct dca_provider *dca, ...@@ -171,7 +178,9 @@ static int ioat_dca_remove_requester(struct dca_provider *dca,
return -ENODEV; return -ENODEV;
} }
static u8 ioat_dca_get_tag(struct dca_provider *dca, int cpu) static u8 ioat_dca_get_tag(struct dca_provider *dca,
struct device *dev,
int cpu)
{ {
struct ioat_dca_priv *ioatdca = dca_priv(dca); struct ioat_dca_priv *ioatdca = dca_priv(dca);
int i, apic_id, bit, value; int i, apic_id, bit, value;
...@@ -193,10 +202,26 @@ static u8 ioat_dca_get_tag(struct dca_provider *dca, int cpu) ...@@ -193,10 +202,26 @@ static u8 ioat_dca_get_tag(struct dca_provider *dca, int cpu)
return tag; return tag;
} }
static int ioat_dca_dev_managed(struct dca_provider *dca,
struct device *dev)
{
struct ioat_dca_priv *ioatdca = dca_priv(dca);
struct pci_dev *pdev;
int i;
pdev = to_pci_dev(dev);
for (i = 0; i < ioatdca->max_requesters; i++) {
if (ioatdca->req_slots[i].pdev == pdev)
return 1;
}
return 0;
}
static struct dca_ops ioat_dca_ops = { static struct dca_ops ioat_dca_ops = {
.add_requester = ioat_dca_add_requester, .add_requester = ioat_dca_add_requester,
.remove_requester = ioat_dca_remove_requester, .remove_requester = ioat_dca_remove_requester,
.get_tag = ioat_dca_get_tag, .get_tag = ioat_dca_get_tag,
.dev_managed = ioat_dca_dev_managed,
}; };
...@@ -207,6 +232,8 @@ struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) ...@@ -207,6 +232,8 @@ struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
u8 *tag_map = NULL; u8 *tag_map = NULL;
int i; int i;
int err; int err;
u8 version;
u8 max_requesters;
if (!system_has_dca_enabled(pdev)) if (!system_has_dca_enabled(pdev))
return NULL; return NULL;
...@@ -237,15 +264,20 @@ struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) ...@@ -237,15 +264,20 @@ struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
if (tag_map == NULL) if (tag_map == NULL)
return NULL; return NULL;
version = readb(iobase + IOAT_VER_OFFSET);
if (version == IOAT_VER_3_0)
max_requesters = IOAT3_DCA_MAX_REQ;
else
max_requesters = IOAT_DCA_MAX_REQ;
dca = alloc_dca_provider(&ioat_dca_ops, dca = alloc_dca_provider(&ioat_dca_ops,
sizeof(*ioatdca) + sizeof(*ioatdca) +
(sizeof(struct ioat_dca_slot) * IOAT_DCA_MAX_REQ)); (sizeof(struct ioat_dca_slot) * max_requesters));
if (!dca) if (!dca)
return NULL; return NULL;
ioatdca = dca_priv(dca); ioatdca = dca_priv(dca);
ioatdca->max_requesters = IOAT_DCA_MAX_REQ; ioatdca->max_requesters = max_requesters;
ioatdca->dca_base = iobase + 0x54; ioatdca->dca_base = iobase + 0x54;
/* copy over the APIC ID to DCA tag mapping */ /* copy over the APIC ID to DCA tag mapping */
...@@ -323,11 +355,13 @@ static int ioat2_dca_remove_requester(struct dca_provider *dca, ...@@ -323,11 +355,13 @@ static int ioat2_dca_remove_requester(struct dca_provider *dca,
return -ENODEV; return -ENODEV;
} }
static u8 ioat2_dca_get_tag(struct dca_provider *dca, int cpu) static u8 ioat2_dca_get_tag(struct dca_provider *dca,
struct device *dev,
int cpu)
{ {
u8 tag; u8 tag;
tag = ioat_dca_get_tag(dca, cpu); tag = ioat_dca_get_tag(dca, dev, cpu);
tag = (~tag) & 0x1F; tag = (~tag) & 0x1F;
return tag; return tag;
} }
...@@ -336,6 +370,7 @@ static struct dca_ops ioat2_dca_ops = { ...@@ -336,6 +370,7 @@ static struct dca_ops ioat2_dca_ops = {
.add_requester = ioat2_dca_add_requester, .add_requester = ioat2_dca_add_requester,
.remove_requester = ioat2_dca_remove_requester, .remove_requester = ioat2_dca_remove_requester,
.get_tag = ioat2_dca_get_tag, .get_tag = ioat2_dca_get_tag,
.dev_managed = ioat_dca_dev_managed,
}; };
static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset) static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
...@@ -425,3 +460,198 @@ struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase) ...@@ -425,3 +460,198 @@ struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
return dca; return dca;
} }
static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev)
{
struct ioat_dca_priv *ioatdca = dca_priv(dca);
struct pci_dev *pdev;
int i;
u16 id;
u16 global_req_table;
/* This implementation only supports PCI-Express */
if (dev->bus != &pci_bus_type)
return -ENODEV;
pdev = to_pci_dev(dev);
id = dcaid_from_pcidev(pdev);
if (ioatdca->requester_count == ioatdca->max_requesters)
return -ENODEV;
for (i = 0; i < ioatdca->max_requesters; i++) {
if (ioatdca->req_slots[i].pdev == NULL) {
/* found an empty slot */
ioatdca->requester_count++;
ioatdca->req_slots[i].pdev = pdev;
ioatdca->req_slots[i].rid = id;
global_req_table =
readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
writel(id | IOAT_DCA_GREQID_VALID,
ioatdca->iobase + global_req_table + (i * 4));
return i;
}
}
/* Error, ioatdma->requester_count is out of whack */
return -EFAULT;
}
static int ioat3_dca_remove_requester(struct dca_provider *dca,
struct device *dev)
{
struct ioat_dca_priv *ioatdca = dca_priv(dca);
struct pci_dev *pdev;
int i;
u16 global_req_table;
/* This implementation only supports PCI-Express */
if (dev->bus != &pci_bus_type)
return -ENODEV;
pdev = to_pci_dev(dev);
for (i = 0; i < ioatdca->max_requesters; i++) {
if (ioatdca->req_slots[i].pdev == pdev) {
global_req_table =
readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
writel(0, ioatdca->iobase + global_req_table + (i * 4));
ioatdca->req_slots[i].pdev = NULL;
ioatdca->req_slots[i].rid = 0;
ioatdca->requester_count--;
return i;
}
}
return -ENODEV;
}
static u8 ioat3_dca_get_tag(struct dca_provider *dca,
struct device *dev,
int cpu)
{
u8 tag;
struct ioat_dca_priv *ioatdca = dca_priv(dca);
int i, apic_id, bit, value;
u8 entry;
tag = 0;
apic_id = cpu_physical_id(cpu);
for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
entry = ioatdca->tag_map[i];
if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
bit = entry &
~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
value = (apic_id & (1 << bit)) ? 1 : 0;
} else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
value = (apic_id & (1 << bit)) ? 0 : 1;
} else {
value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
}
tag |= (value << i);
}
return tag;
}
static struct dca_ops ioat3_dca_ops = {
.add_requester = ioat3_dca_add_requester,
.remove_requester = ioat3_dca_remove_requester,
.get_tag = ioat3_dca_get_tag,
.dev_managed = ioat_dca_dev_managed,
};
static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
{
int slots = 0;
u32 req;
u16 global_req_table;
global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
if (global_req_table == 0)
return 0;
do {
req = readl(iobase + global_req_table + (slots * sizeof(u32)));
slots++;
} while ((req & IOAT_DCA_GREQID_LASTID) == 0);
return slots;
}
struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
int slots;
int i;
int err;
u16 dca_offset;
u16 csi_fsb_control;
u16 pcie_control;
u8 bit;
union {
u64 full;
struct {
u32 low;
u32 high;
};
} tag_map;
if (!system_has_dca_enabled(pdev))
return NULL;
dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
if (dca_offset == 0)
return NULL;
slots = ioat3_dca_count_dca_slots(iobase, dca_offset);
if (slots == 0)
return NULL;
dca = alloc_dca_provider(&ioat3_dca_ops,
sizeof(*ioatdca)
+ (sizeof(struct ioat_dca_slot) * slots));
if (!dca)
return NULL;
ioatdca = dca_priv(dca);
ioatdca->iobase = iobase;
ioatdca->dca_base = iobase + dca_offset;
ioatdca->max_requesters = slots;
/* some bios might not know to turn these on */
csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
writew(csi_fsb_control,
ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
}
pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
writew(pcie_control,
ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
}
/* TODO version, compatibility and configuration checks */
/* copy out the APIC to DCA tag map */
tag_map.low =
readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
tag_map.high =
readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
for (i = 0; i < 8; i++) {
bit = tag_map.full >> (8 * i);
ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
}
err = register_dca_provider(dca, &pdev->dev);
if (err) {
free_dca_provider(dca);
return NULL;
}
return dca;
}
This diff is collapsed.
...@@ -27,8 +27,9 @@ ...@@ -27,8 +27,9 @@
#include <linux/dmapool.h> #include <linux/dmapool.h>
#include <linux/cache.h> #include <linux/cache.h>
#include <linux/pci_ids.h> #include <linux/pci_ids.h>
#include <net/tcp.h>
#define IOAT_DMA_VERSION "2.04" #define IOAT_DMA_VERSION "3.30"
enum ioat_interrupt { enum ioat_interrupt {
none = 0, none = 0,
...@@ -40,6 +41,7 @@ enum ioat_interrupt { ...@@ -40,6 +41,7 @@ enum ioat_interrupt {
#define IOAT_LOW_COMPLETION_MASK 0xffffffc0 #define IOAT_LOW_COMPLETION_MASK 0xffffffc0
#define IOAT_DMA_DCA_ANY_CPU ~0 #define IOAT_DMA_DCA_ANY_CPU ~0
#define IOAT_WATCHDOG_PERIOD (2 * HZ)
/** /**
...@@ -62,6 +64,7 @@ struct ioatdma_device { ...@@ -62,6 +64,7 @@ struct ioatdma_device {
struct dma_device common; struct dma_device common;
u8 version; u8 version;
enum ioat_interrupt irq_mode; enum ioat_interrupt irq_mode;
struct delayed_work work;
struct msix_entry msix_entries[4]; struct msix_entry msix_entries[4];
struct ioat_dma_chan *idx[4]; struct ioat_dma_chan *idx[4];
}; };
...@@ -75,6 +78,7 @@ struct ioat_dma_chan { ...@@ -75,6 +78,7 @@ struct ioat_dma_chan {
dma_cookie_t completed_cookie; dma_cookie_t completed_cookie;
unsigned long last_completion; unsigned long last_completion;
unsigned long last_completion_time;
size_t xfercap; /* XFERCAP register value expanded out */ size_t xfercap; /* XFERCAP register value expanded out */
...@@ -82,6 +86,10 @@ struct ioat_dma_chan { ...@@ -82,6 +86,10 @@ struct ioat_dma_chan {
spinlock_t desc_lock; spinlock_t desc_lock;
struct list_head free_desc; struct list_head free_desc;
struct list_head used_desc; struct list_head used_desc;
unsigned long watchdog_completion;
int watchdog_tcp_cookie;
u32 watchdog_last_tcp_cookie;
struct delayed_work work;
int pending; int pending;
int dmacount; int dmacount;
...@@ -98,6 +106,7 @@ struct ioat_dma_chan { ...@@ -98,6 +106,7 @@ struct ioat_dma_chan {
u32 high; u32 high;
}; };
} *completion_virt; } *completion_virt;
unsigned long last_compl_desc_addr_hw;
struct tasklet_struct cleanup_task; struct tasklet_struct cleanup_task;
}; };
...@@ -121,17 +130,34 @@ struct ioat_desc_sw { ...@@ -121,17 +130,34 @@ struct ioat_desc_sw {
struct dma_async_tx_descriptor async_tx; struct dma_async_tx_descriptor async_tx;
}; };
static inline void ioat_set_tcp_copy_break(struct ioatdma_device *dev)
{
#ifdef CONFIG_NET_DMA
switch (dev->version) {
case IOAT_VER_1_2:
case IOAT_VER_3_0:
sysctl_tcp_dma_copybreak = 4096;
break;
case IOAT_VER_2_0:
sysctl_tcp_dma_copybreak = 2048;
break;
}
#endif
}
#if defined(CONFIG_INTEL_IOATDMA) || defined(CONFIG_INTEL_IOATDMA_MODULE) #if defined(CONFIG_INTEL_IOATDMA) || defined(CONFIG_INTEL_IOATDMA_MODULE)
struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev, struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
void __iomem *iobase); void __iomem *iobase);
void ioat_dma_remove(struct ioatdma_device *device); void ioat_dma_remove(struct ioatdma_device *device);
struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
#else #else
#define ioat_dma_probe(pdev, iobase) NULL #define ioat_dma_probe(pdev, iobase) NULL
#define ioat_dma_remove(device) do { } while (0) #define ioat_dma_remove(device) do { } while (0)
#define ioat_dca_init(pdev, iobase) NULL #define ioat_dca_init(pdev, iobase) NULL
#define ioat2_dca_init(pdev, iobase) NULL #define ioat2_dca_init(pdev, iobase) NULL
#define ioat3_dca_init(pdev, iobase) NULL
#endif #endif
#endif /* IOATDMA_H */ #endif /* IOATDMA_H */
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
#define IOAT_PCI_SID 0x8086 #define IOAT_PCI_SID 0x8086
#define IOAT_VER_1_2 0x12 /* Version 1.2 */ #define IOAT_VER_1_2 0x12 /* Version 1.2 */
#define IOAT_VER_2_0 0x20 /* Version 2.0 */ #define IOAT_VER_2_0 0x20 /* Version 2.0 */
#define IOAT_VER_3_0 0x30 /* Version 3.0 */
struct ioat_dma_descriptor { struct ioat_dma_descriptor {
uint32_t size; uint32_t size;
......
...@@ -25,6 +25,10 @@ ...@@ -25,6 +25,10 @@
#define IOAT_PCI_DMACTRL_DMA_EN 0x00000001 #define IOAT_PCI_DMACTRL_DMA_EN 0x00000001
#define IOAT_PCI_DMACTRL_MSI_EN 0x00000002 #define IOAT_PCI_DMACTRL_MSI_EN 0x00000002
#define IOAT_PCI_DEVICE_ID_OFFSET 0x02
#define IOAT_PCI_DMAUNCERRSTS_OFFSET 0x148
#define IOAT_PCI_CHANERRMASK_INT_OFFSET 0x184
/* MMIO Device Registers */ /* MMIO Device Registers */
#define IOAT_CHANCNT_OFFSET 0x00 /* 8-bit */ #define IOAT_CHANCNT_OFFSET 0x00 /* 8-bit */
...@@ -149,7 +153,23 @@ ...@@ -149,7 +153,23 @@
#define IOAT_DCA_GREQID_VALID 0x20000000 #define IOAT_DCA_GREQID_VALID 0x20000000
#define IOAT_DCA_GREQID_LASTID 0x80000000 #define IOAT_DCA_GREQID_LASTID 0x80000000
#define IOAT3_CSI_CAPABILITY_OFFSET 0x08
#define IOAT3_CSI_CAPABILITY_PREFETCH 0x1
#define IOAT3_PCI_CAPABILITY_OFFSET 0x0A
#define IOAT3_PCI_CAPABILITY_MEMWR 0x1
#define IOAT3_CSI_CONTROL_OFFSET 0x0C
#define IOAT3_CSI_CONTROL_PREFETCH 0x1
#define IOAT3_PCI_CONTROL_OFFSET 0x0E
#define IOAT3_PCI_CONTROL_MEMWR 0x1
#define IOAT3_APICID_TAG_MAP_OFFSET 0x10
#define IOAT3_APICID_TAG_MAP_OFFSET_LOW 0x10
#define IOAT3_APICID_TAG_MAP_OFFSET_HIGH 0x14
#define IOAT3_DCA_GREQID_OFFSET 0x02
#define IOAT1_CHAINADDR_OFFSET 0x0C /* 64-bit Descriptor Chain Address Register */ #define IOAT1_CHAINADDR_OFFSET 0x0C /* 64-bit Descriptor Chain Address Register */
#define IOAT2_CHAINADDR_OFFSET 0x10 /* 64-bit Descriptor Chain Address Register */ #define IOAT2_CHAINADDR_OFFSET 0x10 /* 64-bit Descriptor Chain Address Register */
......
...@@ -82,17 +82,24 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, ...@@ -82,17 +82,24 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
struct device *dev = struct device *dev =
&iop_chan->device->pdev->dev; &iop_chan->device->pdev->dev;
u32 len = unmap->unmap_len; u32 len = unmap->unmap_len;
u32 src_cnt = unmap->unmap_src_cnt; enum dma_ctrl_flags flags = desc->async_tx.flags;
dma_addr_t addr = iop_desc_get_dest_addr(unmap, u32 src_cnt;
iop_chan); dma_addr_t addr;
dma_unmap_page(dev, addr, len, DMA_FROM_DEVICE); if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
while (src_cnt--) { addr = iop_desc_get_dest_addr(unmap, iop_chan);
addr = iop_desc_get_src_addr(unmap, dma_unmap_page(dev, addr, len, DMA_FROM_DEVICE);
iop_chan, }
src_cnt);
dma_unmap_page(dev, addr, len, if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
DMA_TO_DEVICE); src_cnt = unmap->unmap_src_cnt;
while (src_cnt--) {
addr = iop_desc_get_src_addr(unmap,
iop_chan,
src_cnt);
dma_unmap_page(dev, addr, len,
DMA_TO_DEVICE);
}
} }
desc->group_head = NULL; desc->group_head = NULL;
} }
...@@ -366,8 +373,8 @@ retry: ...@@ -366,8 +373,8 @@ retry:
if (!retry++) if (!retry++)
goto retry; goto retry;
/* try to free some slots if the allocation fails */ /* perform direct reclaim if the allocation fails */
tasklet_schedule(&iop_chan->irq_tasklet); __iop_adma_slot_cleanup(iop_chan);
return NULL; return NULL;
} }
...@@ -443,8 +450,18 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) ...@@ -443,8 +450,18 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
/* returns the number of allocated descriptors */ /**
static int iop_adma_alloc_chan_resources(struct dma_chan *chan) * iop_adma_alloc_chan_resources - returns the number of allocated descriptors
* @chan - allocate descriptor resources for this channel
* @client - current client requesting the channel be ready for requests
*
* Note: We keep the slots for 1 operation on iop_chan->chain at all times. To
* avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
* greater than 2x the number slots needed to satisfy a device->max_xor
* request.
* */
static int iop_adma_alloc_chan_resources(struct dma_chan *chan,
struct dma_client *client)
{ {
char *hw_desc; char *hw_desc;
int idx; int idx;
...@@ -838,7 +855,7 @@ static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device) ...@@ -838,7 +855,7 @@ static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device)
dma_chan = container_of(device->common.channels.next, dma_chan = container_of(device->common.channels.next,
struct dma_chan, struct dma_chan,
device_node); device_node);
if (iop_adma_alloc_chan_resources(dma_chan) < 1) { if (iop_adma_alloc_chan_resources(dma_chan, NULL) < 1) {
err = -ENODEV; err = -ENODEV;
goto out; goto out;
} }
...@@ -936,7 +953,7 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device) ...@@ -936,7 +953,7 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
dma_chan = container_of(device->common.channels.next, dma_chan = container_of(device->common.channels.next,
struct dma_chan, struct dma_chan,
device_node); device_node);
if (iop_adma_alloc_chan_resources(dma_chan) < 1) { if (iop_adma_alloc_chan_resources(dma_chan, NULL) < 1) {
err = -ENODEV; err = -ENODEV;
goto out; goto out;
} }
...@@ -1387,6 +1404,8 @@ static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) ...@@ -1387,6 +1404,8 @@ static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
spin_unlock_bh(&iop_chan->lock); spin_unlock_bh(&iop_chan->lock);
} }
MODULE_ALIAS("platform:iop-adma");
static struct platform_driver iop_adma_driver = { static struct platform_driver iop_adma_driver = {
.probe = iop_adma_probe, .probe = iop_adma_probe,
.remove = iop_adma_remove, .remove = iop_adma_remove,
......
This diff is collapsed.
/*
* Copyright (C) 2007, 2008, Marvell International Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef MV_XOR_H
#define MV_XOR_H
#include <linux/types.h>
#include <linux/io.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#define USE_TIMER
#define MV_XOR_SLOT_SIZE 64
#define MV_XOR_THRESHOLD 1
#define XOR_OPERATION_MODE_XOR 0
#define XOR_OPERATION_MODE_MEMCPY 2
#define XOR_OPERATION_MODE_MEMSET 4
#define XOR_CURR_DESC(chan) (chan->mmr_base + 0x210 + (chan->idx * 4))
#define XOR_NEXT_DESC(chan) (chan->mmr_base + 0x200 + (chan->idx * 4))
#define XOR_BYTE_COUNT(chan) (chan->mmr_base + 0x220 + (chan->idx * 4))
#define XOR_DEST_POINTER(chan) (chan->mmr_base + 0x2B0 + (chan->idx * 4))
#define XOR_BLOCK_SIZE(chan) (chan->mmr_base + 0x2C0 + (chan->idx * 4))
#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_base + 0x2E0)
#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_base + 0x2E4)
#define XOR_CONFIG(chan) (chan->mmr_base + 0x10 + (chan->idx * 4))
#define XOR_ACTIVATION(chan) (chan->mmr_base + 0x20 + (chan->idx * 4))
#define XOR_INTR_CAUSE(chan) (chan->mmr_base + 0x30)
#define XOR_INTR_MASK(chan) (chan->mmr_base + 0x40)
#define XOR_ERROR_CAUSE(chan) (chan->mmr_base + 0x50)
#define XOR_ERROR_ADDR(chan) (chan->mmr_base + 0x60)
#define XOR_INTR_MASK_VALUE 0x3F5
#define WINDOW_BASE(w) (0x250 + ((w) << 2))
#define WINDOW_SIZE(w) (0x270 + ((w) << 2))
#define WINDOW_REMAP_HIGH(w) (0x290 + ((w) << 2))
#define WINDOW_BAR_ENABLE(chan) (0x240 + ((chan) << 2))
struct mv_xor_shared_private {
void __iomem *xor_base;
void __iomem *xor_high_base;
};
/**
* struct mv_xor_device - internal representation of a XOR device
* @pdev: Platform device
* @id: HW XOR Device selector
* @dma_desc_pool: base of DMA descriptor region (DMA address)
* @dma_desc_pool_virt: base of DMA descriptor region (CPU address)
* @common: embedded struct dma_device
*/
struct mv_xor_device {
struct platform_device *pdev;
int id;
dma_addr_t dma_desc_pool;
void *dma_desc_pool_virt;
struct dma_device common;
struct mv_xor_shared_private *shared;
};
/**
* struct mv_xor_chan - internal representation of a XOR channel
* @pending: allows batching of hardware operations
* @completed_cookie: identifier for the most recently completed operation
* @lock: serializes enqueue/dequeue operations to the descriptors pool
* @mmr_base: memory mapped register base
* @idx: the index of the xor channel
* @chain: device chain view of the descriptors
* @completed_slots: slots completed by HW but still need to be acked
* @device: parent device
* @common: common dmaengine channel object members
* @last_used: place holder for allocation to continue from where it left off
* @all_slots: complete domain of slots usable by the channel
* @slots_allocated: records the actual size of the descriptor slot pool
* @irq_tasklet: bottom half where mv_xor_slot_cleanup runs
*/
struct mv_xor_chan {
int pending;
dma_cookie_t completed_cookie;
spinlock_t lock; /* protects the descriptor slot pool */
void __iomem *mmr_base;
unsigned int idx;
enum dma_transaction_type current_type;
struct list_head chain;
struct list_head completed_slots;
struct mv_xor_device *device;
struct dma_chan common;
struct mv_xor_desc_slot *last_used;
struct list_head all_slots;
int slots_allocated;
struct tasklet_struct irq_tasklet;
#ifdef USE_TIMER
unsigned long cleanup_time;
u32 current_on_last_cleanup;
dma_cookie_t is_complete_cookie;
#endif
};
/**
* struct mv_xor_desc_slot - software descriptor
* @slot_node: node on the mv_xor_chan.all_slots list
* @chain_node: node on the mv_xor_chan.chain list
* @completed_node: node on the mv_xor_chan.completed_slots list
* @hw_desc: virtual address of the hardware descriptor chain
* @phys: hardware address of the hardware descriptor chain
* @group_head: first operation in a transaction
* @slot_cnt: total slots used in an transaction (group of operations)
* @slots_per_op: number of slots per operation
* @idx: pool index
* @unmap_src_cnt: number of xor sources
* @unmap_len: transaction bytecount
* @async_tx: support for the async_tx api
* @group_list: list of slots that make up a multi-descriptor transaction
* for example transfer lengths larger than the supported hw max
* @xor_check_result: result of zero sum
* @crc32_result: result crc calculation
*/
struct mv_xor_desc_slot {
struct list_head slot_node;
struct list_head chain_node;
struct list_head completed_node;
enum dma_transaction_type type;
void *hw_desc;
struct mv_xor_desc_slot *group_head;
u16 slot_cnt;
u16 slots_per_op;
u16 idx;
u16 unmap_src_cnt;
u32 value;
size_t unmap_len;
struct dma_async_tx_descriptor async_tx;
union {
u32 *xor_check_result;
u32 *crc32_result;
};
#ifdef USE_TIMER
unsigned long arrival_time;
struct timer_list timeout;
#endif
};
/* This structure describes XOR descriptor size 64bytes */
struct mv_xor_desc {
u32 status; /* descriptor execution status */
u32 crc32_result; /* result of CRC-32 calculation */
u32 desc_command; /* type of operation to be carried out */
u32 phy_next_desc; /* next descriptor address pointer */
u32 byte_count; /* size of src/dst blocks in bytes */
u32 phy_dest_addr; /* destination block address */
u32 phy_src_addr[8]; /* source block addresses */
u32 reserved0;
u32 reserved1;
};
#define to_mv_sw_desc(addr_hw_desc) \
container_of(addr_hw_desc, struct mv_xor_desc_slot, hw_desc)
#define mv_hw_desc_slot_idx(hw_desc, idx) \
((void *)(((unsigned long)hw_desc) + ((idx) << 5)))
#define MV_XOR_MIN_BYTE_COUNT (128)
#define XOR_MAX_BYTE_COUNT ((16 * 1024 * 1024) - 1)
#define MV_XOR_MAX_BYTE_COUNT XOR_MAX_BYTE_COUNT
#endif
...@@ -198,17 +198,13 @@ iop_chan_memset_slot_count(size_t len, int *slots_per_op) ...@@ -198,17 +198,13 @@ iop_chan_memset_slot_count(size_t len, int *slots_per_op)
static inline int static inline int
iop_chan_xor_slot_count(size_t len, int src_cnt, int *slots_per_op) iop_chan_xor_slot_count(size_t len, int src_cnt, int *slots_per_op)
{ {
int num_slots; static const char slot_count_table[] = { 1, 2, 2, 2,
/* slots_to_find = 1 for basic descriptor + 1 per 4 sources above 1 2, 3, 3, 3,
* (1 source => 8 bytes) (1 slot => 32 bytes) 3, 4, 4, 4,
*/ 4, 5, 5, 5,
num_slots = 1 + (((src_cnt - 1) << 3) >> 5); };
if (((src_cnt - 1) << 3) & 0x1f) *slots_per_op = slot_count_table[src_cnt - 1];
num_slots++; return *slots_per_op;
*slots_per_op = num_slots;
return num_slots;
} }
#define ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024) #define ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
......
...@@ -260,7 +260,7 @@ static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op) ...@@ -260,7 +260,7 @@ static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op)
static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt, static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt,
int *slots_per_op) int *slots_per_op)
{ {
static const int slot_count_table[] = { 0, static const char slot_count_table[] = {
1, 1, 1, 1, /* 01 - 04 */ 1, 1, 1, 1, /* 01 - 04 */
2, 2, 2, 2, /* 05 - 08 */ 2, 2, 2, 2, /* 05 - 08 */
4, 4, 4, 4, /* 09 - 12 */ 4, 4, 4, 4, /* 09 - 12 */
...@@ -270,7 +270,7 @@ static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt, ...@@ -270,7 +270,7 @@ static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt,
8, 8, 8, 8, /* 25 - 28 */ 8, 8, 8, 8, /* 25 - 28 */
8, 8, 8, 8, /* 29 - 32 */ 8, 8, 8, 8, /* 29 - 32 */
}; };
*slots_per_op = slot_count_table[src_cnt]; *slots_per_op = slot_count_table[src_cnt - 1];
return *slots_per_op; return *slots_per_op;
} }
......
/*
* Marvell XOR platform device data definition file.
*/
#ifndef __ASM_PLAT_ORION_MV_XOR_H
#define __ASM_PLAT_ORION_MV_XOR_H
#include <linux/dmaengine.h>
#include <linux/mbus.h>
#define MV_XOR_SHARED_NAME "mv_xor_shared"
#define MV_XOR_NAME "mv_xor"
struct mbus_dram_target_info;
struct mv_xor_platform_shared_data {
struct mbus_dram_target_info *dram;
};
struct mv_xor_platform_data {
struct platform_device *shared;
int hw_id;
dma_cap_mask_t cap_mask;
size_t pool_size;
};
#endif
...@@ -30,4 +30,20 @@ ...@@ -30,4 +30,20 @@
#define GPIO_PIN_PD(N) (GPIO_PIOD_BASE + (N)) #define GPIO_PIN_PD(N) (GPIO_PIOD_BASE + (N))
#define GPIO_PIN_PE(N) (GPIO_PIOE_BASE + (N)) #define GPIO_PIN_PE(N) (GPIO_PIOE_BASE + (N))
/*
* DMAC peripheral hardware handshaking interfaces, used with dw_dmac
*/
#define DMAC_MCI_RX 0
#define DMAC_MCI_TX 1
#define DMAC_DAC_TX 2
#define DMAC_AC97_A_RX 3
#define DMAC_AC97_A_TX 4
#define DMAC_AC97_B_RX 5
#define DMAC_AC97_B_TX 6
#define DMAC_DMAREQ_0 7
#define DMAC_DMAREQ_1 8
#define DMAC_DMAREQ_2 9
#define DMAC_DMAREQ_3 10
#endif /* __ASM_ARCH_AT32AP700X_H__ */ #endif /* __ASM_ARCH_AT32AP700X_H__ */
...@@ -101,21 +101,14 @@ async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx, ...@@ -101,21 +101,14 @@ async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
/** /**
* async_tx_sync_epilog - actions to take if an operation is run synchronously * async_tx_sync_epilog - actions to take if an operation is run synchronously
* @flags: async_tx flags
* @depend_tx: transaction depends on depend_tx
* @cb_fn: function to call when the transaction completes * @cb_fn: function to call when the transaction completes
* @cb_fn_param: parameter to pass to the callback routine * @cb_fn_param: parameter to pass to the callback routine
*/ */
static inline void static inline void
async_tx_sync_epilog(unsigned long flags, async_tx_sync_epilog(dma_async_tx_callback cb_fn, void *cb_fn_param)
struct dma_async_tx_descriptor *depend_tx,
dma_async_tx_callback cb_fn, void *cb_fn_param)
{ {
if (cb_fn) if (cb_fn)
cb_fn(cb_fn_param); cb_fn(cb_fn_param);
if (depend_tx && (flags & ASYNC_TX_DEP_ACK))
async_tx_ack(depend_tx);
} }
void void
...@@ -152,4 +145,6 @@ struct dma_async_tx_descriptor * ...@@ -152,4 +145,6 @@ struct dma_async_tx_descriptor *
async_trigger_callback(enum async_tx_flags flags, async_trigger_callback(enum async_tx_flags flags,
struct dma_async_tx_descriptor *depend_tx, struct dma_async_tx_descriptor *depend_tx,
dma_async_tx_callback cb_fn, void *cb_fn_param); dma_async_tx_callback cb_fn, void *cb_fn_param);
void async_tx_quiesce(struct dma_async_tx_descriptor **tx);
#endif /* _ASYNC_TX_H_ */ #endif /* _ASYNC_TX_H_ */
...@@ -10,6 +10,7 @@ void dca_unregister_notify(struct notifier_block *nb); ...@@ -10,6 +10,7 @@ void dca_unregister_notify(struct notifier_block *nb);
#define DCA_PROVIDER_REMOVE 0x0002 #define DCA_PROVIDER_REMOVE 0x0002
struct dca_provider { struct dca_provider {
struct list_head node;
struct dca_ops *ops; struct dca_ops *ops;
struct device *cd; struct device *cd;
int id; int id;
...@@ -18,7 +19,9 @@ struct dca_provider { ...@@ -18,7 +19,9 @@ struct dca_provider {
struct dca_ops { struct dca_ops {
int (*add_requester) (struct dca_provider *, struct device *); int (*add_requester) (struct dca_provider *, struct device *);
int (*remove_requester) (struct dca_provider *, struct device *); int (*remove_requester) (struct dca_provider *, struct device *);
u8 (*get_tag) (struct dca_provider *, int cpu); u8 (*get_tag) (struct dca_provider *, struct device *,
int cpu);
int (*dev_managed) (struct dca_provider *, struct device *);
}; };
struct dca_provider *alloc_dca_provider(struct dca_ops *ops, int priv_size); struct dca_provider *alloc_dca_provider(struct dca_ops *ops, int priv_size);
...@@ -32,9 +35,11 @@ static inline void *dca_priv(struct dca_provider *dca) ...@@ -32,9 +35,11 @@ static inline void *dca_priv(struct dca_provider *dca)
} }
/* Requester API */ /* Requester API */
#define DCA_GET_TAG_TWO_ARGS
int dca_add_requester(struct device *dev); int dca_add_requester(struct device *dev);
int dca_remove_requester(struct device *dev); int dca_remove_requester(struct device *dev);
u8 dca_get_tag(int cpu); u8 dca_get_tag(int cpu);
u8 dca3_get_tag(struct device *dev, int cpu);
/* internal stuff */ /* internal stuff */
int __init dca_sysfs_init(void); int __init dca_sysfs_init(void);
......
...@@ -89,10 +89,23 @@ enum dma_transaction_type { ...@@ -89,10 +89,23 @@ enum dma_transaction_type {
DMA_MEMSET, DMA_MEMSET,
DMA_MEMCPY_CRC32C, DMA_MEMCPY_CRC32C,
DMA_INTERRUPT, DMA_INTERRUPT,
DMA_SLAVE,
}; };
/* last transaction type for creation of the capabilities mask */ /* last transaction type for creation of the capabilities mask */
#define DMA_TX_TYPE_END (DMA_INTERRUPT + 1) #define DMA_TX_TYPE_END (DMA_SLAVE + 1)
/**
* enum dma_slave_width - DMA slave register access width.
* @DMA_SLAVE_WIDTH_8BIT: Do 8-bit slave register accesses
* @DMA_SLAVE_WIDTH_16BIT: Do 16-bit slave register accesses
* @DMA_SLAVE_WIDTH_32BIT: Do 32-bit slave register accesses
*/
enum dma_slave_width {
DMA_SLAVE_WIDTH_8BIT,
DMA_SLAVE_WIDTH_16BIT,
DMA_SLAVE_WIDTH_32BIT,
};
/** /**
* enum dma_ctrl_flags - DMA flags to augment operation preparation, * enum dma_ctrl_flags - DMA flags to augment operation preparation,
...@@ -102,10 +115,14 @@ enum dma_transaction_type { ...@@ -102,10 +115,14 @@ enum dma_transaction_type {
* @DMA_CTRL_ACK - the descriptor cannot be reused until the client * @DMA_CTRL_ACK - the descriptor cannot be reused until the client
* acknowledges receipt, i.e. has has a chance to establish any * acknowledges receipt, i.e. has has a chance to establish any
* dependency chains * dependency chains
* @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s)
* @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s)
*/ */
enum dma_ctrl_flags { enum dma_ctrl_flags {
DMA_PREP_INTERRUPT = (1 << 0), DMA_PREP_INTERRUPT = (1 << 0),
DMA_CTRL_ACK = (1 << 1), DMA_CTRL_ACK = (1 << 1),
DMA_COMPL_SKIP_SRC_UNMAP = (1 << 2),
DMA_COMPL_SKIP_DEST_UNMAP = (1 << 3),
}; };
/** /**
...@@ -114,6 +131,32 @@ enum dma_ctrl_flags { ...@@ -114,6 +131,32 @@ enum dma_ctrl_flags {
*/ */
typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t; typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
/**
* struct dma_slave - Information about a DMA slave
* @dev: device acting as DMA slave
* @dma_dev: required DMA master device. If non-NULL, the client can not be
* bound to other masters than this.
* @tx_reg: physical address of data register used for
* memory-to-peripheral transfers
* @rx_reg: physical address of data register used for
* peripheral-to-memory transfers
* @reg_width: peripheral register width
*
* If dma_dev is non-NULL, the client can not be bound to other DMA
* masters than the one corresponding to this device. The DMA master
* driver may use this to determine if there is controller-specific
* data wrapped around this struct. Drivers of platform code that sets
* the dma_dev field must therefore make sure to use an appropriate
* controller-specific dma slave structure wrapping this struct.
*/
struct dma_slave {
struct device *dev;
struct device *dma_dev;
dma_addr_t tx_reg;
dma_addr_t rx_reg;
enum dma_slave_width reg_width;
};
/** /**
* struct dma_chan_percpu - the per-CPU part of struct dma_chan * struct dma_chan_percpu - the per-CPU part of struct dma_chan
* @refcount: local_t used for open-coded "bigref" counting * @refcount: local_t used for open-coded "bigref" counting
...@@ -139,6 +182,7 @@ struct dma_chan_percpu { ...@@ -139,6 +182,7 @@ struct dma_chan_percpu {
* @rcu: the DMA channel's RCU head * @rcu: the DMA channel's RCU head
* @device_node: used to add this to the device chan list * @device_node: used to add this to the device chan list
* @local: per-cpu pointer to a struct dma_chan_percpu * @local: per-cpu pointer to a struct dma_chan_percpu
* @client-count: how many clients are using this channel
*/ */
struct dma_chan { struct dma_chan {
struct dma_device *device; struct dma_device *device;
...@@ -154,6 +198,7 @@ struct dma_chan { ...@@ -154,6 +198,7 @@ struct dma_chan {
struct list_head device_node; struct list_head device_node;
struct dma_chan_percpu *local; struct dma_chan_percpu *local;
int client_count;
}; };
#define to_dma_chan(p) container_of(p, struct dma_chan, dev) #define to_dma_chan(p) container_of(p, struct dma_chan, dev)
...@@ -202,11 +247,14 @@ typedef enum dma_state_client (*dma_event_callback) (struct dma_client *client, ...@@ -202,11 +247,14 @@ typedef enum dma_state_client (*dma_event_callback) (struct dma_client *client,
* @event_callback: func ptr to call when something happens * @event_callback: func ptr to call when something happens
* @cap_mask: only return channels that satisfy the requested capabilities * @cap_mask: only return channels that satisfy the requested capabilities
* a value of zero corresponds to any capability * a value of zero corresponds to any capability
* @slave: data for preparing slave transfer. Must be non-NULL iff the
* DMA_SLAVE capability is requested.
* @global_node: list_head for global dma_client_list * @global_node: list_head for global dma_client_list
*/ */
struct dma_client { struct dma_client {
dma_event_callback event_callback; dma_event_callback event_callback;
dma_cap_mask_t cap_mask; dma_cap_mask_t cap_mask;
struct dma_slave *slave;
struct list_head global_node; struct list_head global_node;
}; };
...@@ -263,6 +311,8 @@ struct dma_async_tx_descriptor { ...@@ -263,6 +311,8 @@ struct dma_async_tx_descriptor {
* @device_prep_dma_zero_sum: prepares a zero_sum operation * @device_prep_dma_zero_sum: prepares a zero_sum operation
* @device_prep_dma_memset: prepares a memset operation * @device_prep_dma_memset: prepares a memset operation
* @device_prep_dma_interrupt: prepares an end of chain interrupt operation * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
* @device_prep_slave_sg: prepares a slave dma operation
* @device_terminate_all: terminate all pending operations
* @device_issue_pending: push pending transactions to hardware * @device_issue_pending: push pending transactions to hardware
*/ */
struct dma_device { struct dma_device {
...@@ -279,7 +329,8 @@ struct dma_device { ...@@ -279,7 +329,8 @@ struct dma_device {
int dev_id; int dev_id;
struct device *dev; struct device *dev;
int (*device_alloc_chan_resources)(struct dma_chan *chan); int (*device_alloc_chan_resources)(struct dma_chan *chan,
struct dma_client *client);
void (*device_free_chan_resources)(struct dma_chan *chan); void (*device_free_chan_resources)(struct dma_chan *chan);
struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
...@@ -297,6 +348,12 @@ struct dma_device { ...@@ -297,6 +348,12 @@ struct dma_device {
struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)( struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
struct dma_chan *chan, unsigned long flags); struct dma_chan *chan, unsigned long flags);
struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_data_direction direction,
unsigned long flags);
void (*device_terminate_all)(struct dma_chan *chan);
enum dma_status (*device_is_tx_complete)(struct dma_chan *chan, enum dma_status (*device_is_tx_complete)(struct dma_chan *chan,
dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t cookie, dma_cookie_t *last,
dma_cookie_t *used); dma_cookie_t *used);
...@@ -318,16 +375,14 @@ dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan, ...@@ -318,16 +375,14 @@ dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
struct dma_chan *chan); struct dma_chan *chan);
static inline void static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
async_tx_ack(struct dma_async_tx_descriptor *tx)
{ {
tx->flags |= DMA_CTRL_ACK; tx->flags |= DMA_CTRL_ACK;
} }
static inline int static inline bool async_tx_test_ack(struct dma_async_tx_descriptor *tx)
async_tx_test_ack(struct dma_async_tx_descriptor *tx)
{ {
return tx->flags & DMA_CTRL_ACK; return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;
} }
#define first_dma_cap(mask) __first_dma_cap(&(mask)) #define first_dma_cap(mask) __first_dma_cap(&(mask))
......
/*
* Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
* AVR32 systems.)
*
* Copyright (C) 2007 Atmel Corporation
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef DW_DMAC_H
#define DW_DMAC_H
#include <linux/dmaengine.h>
/**
* struct dw_dma_platform_data - Controller configuration parameters
* @nr_channels: Number of channels supported by hardware (max 8)
*/
struct dw_dma_platform_data {
unsigned int nr_channels;
};
/**
* struct dw_dma_slave - Controller-specific information about a slave
* @slave: Generic information about the slave
* @ctl_lo: Platform-specific initializer for the CTL_LO register
* @cfg_hi: Platform-specific initializer for the CFG_HI register
* @cfg_lo: Platform-specific initializer for the CFG_LO register
*/
struct dw_dma_slave {
struct dma_slave slave;
u32 cfg_hi;
u32 cfg_lo;
};
/* Platform-configurable bits in CFG_HI */
#define DWC_CFGH_FCMODE (1 << 0)
#define DWC_CFGH_FIFO_MODE (1 << 1)
#define DWC_CFGH_PROTCTL(x) ((x) << 2)
#define DWC_CFGH_SRC_PER(x) ((x) << 7)
#define DWC_CFGH_DST_PER(x) ((x) << 11)
/* Platform-configurable bits in CFG_LO */
#define DWC_CFGL_PRIO(x) ((x) << 5) /* priority */
#define DWC_CFGL_LOCK_CH_XFER (0 << 12) /* scope of LOCK_CH */
#define DWC_CFGL_LOCK_CH_BLOCK (1 << 12)
#define DWC_CFGL_LOCK_CH_XACT (2 << 12)
#define DWC_CFGL_LOCK_BUS_XFER (0 << 14) /* scope of LOCK_BUS */
#define DWC_CFGL_LOCK_BUS_BLOCK (1 << 14)
#define DWC_CFGL_LOCK_BUS_XACT (2 << 14)
#define DWC_CFGL_LOCK_CH (1 << 15) /* channel lockout */
#define DWC_CFGL_LOCK_BUS (1 << 16) /* busmaster lockout */
#define DWC_CFGL_HS_DST_POL (1 << 18) /* dst handshake active low */
#define DWC_CFGL_HS_SRC_POL (1 << 19) /* src handshake active low */
static inline struct dw_dma_slave *to_dw_dma_slave(struct dma_slave *slave)
{
return container_of(slave, struct dw_dma_slave, slave);
}
#endif /* DW_DMAC_H */
...@@ -2371,6 +2371,14 @@ ...@@ -2371,6 +2371,14 @@
#define PCI_DEVICE_ID_INTEL_ICH9_7 0x2916 #define PCI_DEVICE_ID_INTEL_ICH9_7 0x2916
#define PCI_DEVICE_ID_INTEL_ICH9_8 0x2918 #define PCI_DEVICE_ID_INTEL_ICH9_8 0x2918
#define PCI_DEVICE_ID_INTEL_82855PM_HB 0x3340 #define PCI_DEVICE_ID_INTEL_82855PM_HB 0x3340
#define PCI_DEVICE_ID_INTEL_IOAT_TBG4 0x3429
#define PCI_DEVICE_ID_INTEL_IOAT_TBG5 0x342a
#define PCI_DEVICE_ID_INTEL_IOAT_TBG6 0x342b
#define PCI_DEVICE_ID_INTEL_IOAT_TBG7 0x342c
#define PCI_DEVICE_ID_INTEL_IOAT_TBG0 0x3430
#define PCI_DEVICE_ID_INTEL_IOAT_TBG1 0x3431
#define PCI_DEVICE_ID_INTEL_IOAT_TBG2 0x3432
#define PCI_DEVICE_ID_INTEL_IOAT_TBG3 0x3433
#define PCI_DEVICE_ID_INTEL_82830_HB 0x3575 #define PCI_DEVICE_ID_INTEL_82830_HB 0x3575
#define PCI_DEVICE_ID_INTEL_82830_CGC 0x3577 #define PCI_DEVICE_ID_INTEL_82830_CGC 0x3577
#define PCI_DEVICE_ID_INTEL_82855GM_HB 0x3580 #define PCI_DEVICE_ID_INTEL_82855GM_HB 0x3580
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
#define NET_DMA_DEFAULT_COPYBREAK 4096 #define NET_DMA_DEFAULT_COPYBREAK 4096
int sysctl_tcp_dma_copybreak = NET_DMA_DEFAULT_COPYBREAK; int sysctl_tcp_dma_copybreak = NET_DMA_DEFAULT_COPYBREAK;
EXPORT_SYMBOL(sysctl_tcp_dma_copybreak);
/** /**
* dma_skb_copy_datagram_iovec - Copy a datagram to an iovec. * dma_skb_copy_datagram_iovec - Copy a datagram to an iovec.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment