/*
* kernel: 4.5.2
*/
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/blkdev.h>
#include <linux/kthread.h>
#include <linux/interrupt.h> // needed in 4.3.3
#define TEST_PCIE_DEV_NAME "test_pcie"
//#define PCI_VENDOR_ID_XILINX 0x10EE /* already defined in <linux/pci_ids.h> The default value, 10EEh, is the Vendor ID for Xilinx. */
#define TEST_PCI_DEVICE_ID_XILINX 0x7033 // !!!check here! /* the default value is 70<link speed><link width>h */
#define TEST_SSD_DEV_NAME "test_ssd"
#define TEST_SSD_PARTITONS 1
#define TEST_SSD_MINORS 1
#define TEST_DEV_MEM_SZ (1 << 22) // 4MB
#define KTHREAD_NAME "test_kthread_fn"
#define COHERENT_DMA_BUF_SZ (1 << 22) // !!!4MB; no less than 1MB
#define TEST_DMA_BUF_MAX_NUM 320 // !!!
/* ZC706 AXI Address */
#define DDR3_BASE_ADDR (0x0 + 0x400000) // !!!!!!
#define AXI_BAR0 0x80800000
#define AXI_BAR1 0x80000000
#define TRANS_BRAM_BASE_ADDR 0x81000000
#define AXI_BAR2 0x40000000 // wb
/* offset from TRANS_BRAM_BASE_ADDR */
#define AXI_PCIE_CTL_OFFSET 0x8000
#define AXI_CDMA_LITE_OFFSET 0xc000
/* AXI to PCIe Base Address Translation Configuration Registers
* offset from AXI_PCIE_CTL_OFFSET
*/
#define AXIBAR2PCIEBAR_0U_OFFSET 0x208
#define AXIBAR2PCIEBAR_0L_OFFSET 0x20c
#define AXIBAR2PCIEBAR_1U_OFFSET 0x210
#define AXIBAR2PCIEBAR_1L_OFFSET 0x214
#define AXIBAR2PCIEBAR_2U_OFFSET 0x218
#define AXIBAR2PCIEBAR_2L_OFFSET 0x21c
/* trans_desc_offset */
/*
#define NXTDESC_PNTR_OFFSET 0x00
#define DESC_SA_OFFSET 0x08
#define DESC_DA_OFFSET 0x10
#define DESC_CTL_OFFSET 0x18
#define DESC_STAT_OFFSET 0x1c
*/
/* cdma_reg_offset
* offset from AXI_CDMA_LITE_OFFSET
*/
#define CDMACR_OFFSET 0x00
#define CDMASR_OFFSET 0x04
#define CURDESC_PNTR_OFFSET 0x08
#define TAILDESC_PNTR_OFFSET 0x10
#define CDMA_SA_OFFSET 0x18
#define CDMA_DA_OFFSET 0x20
#define CDMA_BTT_OFFSET 0x28
struct io_cmd {
struct bio *bio; // !!!
struct scatterlist *scatlist;
dma_addr_t dma_addr; // used by DMA controller of the device
void *kvaddr; // kernel virtual address, used by kernel and driver, especially to deal with data from userspace(__bio_kmap_atomic)
uint32_t len;
};
struct io_que {
struct bio_list bio_lst; // !!!composed of bio, a singly-linked list of bios
struct task_struct *task_s;
struct io_cmd *io_cmd; // above
struct ssd_dev *ssd_dev; // !!!below
spinlock_t lock;
uint8_t volatile is_busy; // origin: unsigned int, DMA busy flag
};
struct ssd_dev {
struct pci_dev *pci_dev;
struct gendisk *disk; // linux/genhd.h
void __iomem *pci_bar; // !!!!!!above, __iomem is needed
struct io_que *dev_que; // !!!above
};
#define TRANS_DESC_ALIGN 0x40 // !!!!!!64
struct trans_desc { // transfer descriptor, according to xapp1171
uint32_t nxt_ptr;
uint32_t reserved0;
uint32_t src_addr;
uint32_t reserved1;
uint32_t dest_addr;
uint32_t reserved2;
uint32_t ctrl;
uint32_t stat;
};
#define __MIN(a, b) ((a) < (b) ? (a) : (b))
/*
static void setup_cmd(struct io_cmd *io_cmd, struct bio *bio, struct io_que *dev_que)
{
io_cmd->bio = bio; // !!!!!!save it until test_bio_complete
}
*/
/*
static int setup_scatter_map(struct ssd_dev *ssd_dev, struct io_cmd *io_cmd, unsigned int const phys_segs)
{
void *kvaddr; // volatile struct scatter_region *
dma_addr_t dma_addr;
// !!!!!!return two params! set here!
kvaddr = dma_alloc_coherent(&ssd_dev->pci_dev->dev, PAGE_SIZE, &dma_addr, GFP_ATOMIC | GFP_DMA);
if (kvaddr == NULL) {
printk("err_dma_pool_alloc\n");
return -ENOMEM;
}
io_cmd->kvaddr = kvaddr;
io_cmd->dma_addr = dma_addr;
io_cmd->len = phys_segs;
return 0;
}
*/
/*
static int setup_scatter_list(struct io_que *dev_que, struct io_cmd *io_cmd, struct bio *bio)
{
//struct ssd_dev *ssd_dev;
struct bio_vec prev_bv, cur_bv; // !!!
struct bvec_iter bvec_iter; // !!!
struct scatterlist *cur_scatlist = NULL;
unsigned int phys_segs, bytes_len = 0;
unsigned char isnt_first_bio_vec = 0u; // !!!
int result = -ENOMEM;
phys_segs = bio_phys_segments(dev_que->ssd_dev->disk->queue, bio); // !!!
io_cmd->scatlist = (struct scatterlist *)kmalloc(sizeof(struct scatterlist) * phys_segs, GFP_ATOMIC | GFP_DMA); // !!!
if (io_cmd->scatlist == NULL) {
printk("err_alloc_scatterlist\n");
goto err_alloc_scatterlist;
}
sg_init_table(io_cmd->scatlist, phys_segs); // !!!lib/scatterlist.c
phys_segs = 0;
memset(&prev_bv, 0, sizeof(struct bio_vec)); // !!!!!!prev_bv need to be initialized
bio_for_each_segment(cur_bv, bio, bvec_iter) { // !!!
if (isnt_first_bio_vec && BIOVEC_PHYS_MERGEABLE(&prev_bv, &cur_bv)) { // !!!BIOVEC_PHYS_MERGEABLE is defined in bio.h
cur_scatlist->length += cur_bv.bv_len; // !!!
} else {
if (isnt_first_bio_vec)
cur_scatlist++;
else
cur_scatlist = io_cmd->scatlist;
sg_set_page(cur_scatlist, cur_bv.bv_page, cur_bv.bv_len, cur_bv.bv_offset); // !!!in <linux/scatterlist.h>
phys_segs++;
}
bytes_len += cur_bv.bv_len; // !!!
prev_bv = cur_bv;
isnt_first_bio_vec = 1u;
}
sg_mark_end(cur_scatlist); // !!!<linux/scatterlist.h>
//ssd_dev = dev_que->ssd_dev;
result = dma_map_sg(&dev_que->ssd_dev->pci_dev->dev, io_cmd->scatlist, phys_segs,
bio_data_dir(io_cmd->bio) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE); // !!!???What's its use?
if (result == 0) {
printk("err_dma_map_sg\n");
goto err_dma_map_sg;
}
result = setup_scatter_map(dev_que->ssd_dev, io_cmd, phys_segs); // above
if (result)
goto err_setup_scatter_map;
bio->bi_iter.bi_sector += (sector_t)(bytes_len >> 9); // !!!it will not be set by the kernel?
bio->bi_iter.bi_idx = bvec_iter.bi_idx; // !!!
return 0;
err_setup_scatter_map:
dma_unmap_sg(&dev_que->ssd_dev->pci_dev->dev, io_cmd->scatlist, phys_segs,
bio_data_dir(io_cmd->bio) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
printk("err_setup_scatter_map\n");
err_dma_map_sg:
kfree(io_cmd->scatlist);
err_alloc_scatterlist:
return -ENOMEM;
}
*/
/*
static void submit_cmd(struct io_que *dev_que) // !!!actually starts a DMA transfer
{
dma_addr_t rq_dma_addr;
struct ssd_dev *ssd_dev;
ssd_dev = dev_que->ssd_dev;
rq_dma_addr = dev_que->io_cmd->dma_addr;
dev_que->is_busy = 1; // !!!!!!busy flag
}
*/
/*
static int make_bio_request(struct io_que *io_que, struct bio *bio)
{
int result = -EBUSY;
setup_cmd(io_que->io_cmd, bio, io_que); // above, has modified io_que->io_cmd
result = setup_scatter_list(io_que, io_que->io_cmd, bio); // above
if (result) {
printk("err_setup_scatter_list\n");
goto err_setup_scatter_list;
}
submit_cmd(io_que); // above
return 0;
err_setup_scatter_list:
return -ENOMEM;
}
*/
static inline void cdma_set_sg_mode(void __iomem *pci_bar)
{
unsigned long val;
val = readl((unsigned char *)pci_bar + AXI_CDMA_LITE_OFFSET + CDMACR_OFFSET);
val |= 0x8;
writel(val, (unsigned char *)pci_bar + AXI_CDMA_LITE_OFFSET + CDMACR_OFFSET);
val = readl((unsigned char *)pci_bar + AXI_CDMA_LITE_OFFSET + CDMACR_OFFSET);
//if ((val >> 3) & 0x1)
//printk("cdma_init_sg success\n");
printk("after init_sg CDMA_CTRL: %lx\n", val);
}
static unsigned char cdma_wait_idle(void __iomem *pci_bar)
{
unsigned long val, cnt = 0;
unsigned char flag;
while (1) {
val = readl((unsigned char *)pci_bar + AXI_CDMA_LITE_OFFSET + CDMASR_OFFSET);
if ((val >> 1) & 0x1) {
if (val & 0x770) {
printk("some errors occur\n");
flag = 1;
} else
flag = 0;
break;
}
cnt++;
if (cnt > 10000) {
printk("err: timeout\n");
flag = 2;
break;
}
}
return flag;
}
static void fill_trans_desc(void *base, uint32_t no, unsigned char is_last,
uint32_t src_addr, uint32_t dest_addr, uint32_t len)
{
struct trans_desc *desc = (struct trans_desc *)((unsigned char *)base + no * TRANS_DESC_ALIGN);
memset(desc, 0, sizeof(struct trans_desc));
desc->nxt_ptr = (is_last == 1 ? AXI_BAR2 : AXI_BAR2 + (no + 1)*TRANS_DESC_ALIGN);
desc->src_addr = src_addr;
desc->dest_addr = dest_addr;
desc->ctrl = len;
}
static void print_desc_stat(void *base, unsigned int no)
{
struct trans_desc *desc = (struct trans_desc *)((unsigned char *)base + no * TRANS_DESC_ALIGN);
printk("no.[%u] trans_desc stat: %x\n", no, desc->stat);
}
static void test_process_bio(struct io_que *io_que, struct bio *bio) // bio's data no greater than 4MB?
{
struct bio_vec bvec;
struct bio_vec *pre_bv;
struct bvec_iter iter;
unsigned int byte_offset = bio->bi_iter.bi_sector << 9; // !!!!!!
unsigned int const dir = bio_data_dir(bio);
//void * const kvaddr = io_que->io_cmd->kvaddr; // get it
//dma_addr_t const dma_addr = io_que->io_cmd->dma_addr; // get it
void **kvaddr;
dma_addr_t *dma_addr;
unsigned int const phys_segs = bio_phys_segments(io_que->ssd_dev->disk->queue, bio);
int h, i, j, k; // !!!signed
unsigned int *bv_len;
void **usr_buf;
void *desc_kvbar;
dma_addr_t desc_dma_bar;
uint32_t upper_segs = __MIN(phys_segs, TEST_DMA_BUF_MAX_NUM);
uint8_t isnt_first_bio_vec = 0u;
//unsigned long val;
//unsigned int dbg_var = 0;
//printk("axi bar1 high 32bits is %x\n",
// readl((unsigned char *)io_que->ssd_dev->pci_bar + AXIBAR2PCIEBAR1_OFFSET_U));
//printk("axi bar1 low 32 bits is %x\n",
// readl((unsigned char *)io_que->ssd_dev->pci_bar + AXIBAR2PCIEBAR1_OFFSET_L));
printk("this bio has %d segs\n", phys_segs); // !!!
kvaddr = (void **)kmalloc(phys_segs * sizeof(void *), GFP_KERNEL); // !!!
if (kvaddr == NULL) {
printk("kmalloc kvaddr failed\n");
goto out_kmalloc_kvaddr;
}
dma_addr = (dma_addr_t *)kmalloc(phys_segs * sizeof(dma_addr_t), GFP_KERNEL); // !!!
if (dma_addr == NULL) {
printk("kmalloc dma_addr failed\n");
goto out_kmalloc_dma_addr;
}
bv_len = (unsigned int *)kmalloc(phys_segs * sizeof(unsigned int), GFP_KERNEL);
if (bv_len == NULL) {
printk("kmalloc bv_len failed\n");
goto out_kmalloc_bv_len;
}
usr_buf = (void **)kmalloc(phys_segs * sizeof(void *), GFP_KERNEL);
if (NULL == usr_buf) {
printk("kmalloc usr_buf failed\n");
goto out_kmalloc_usr_buf;
}
i = 0;
bio_for_each_segment(bvec, bio, iter) {
void *buffer = __bio_kmap_atomic(bio, iter);
unsigned int cur_bv_len = bio_cur_bytes(bio);
if (isnt_first_bio_vec && BIOVEC_PHYS_MERGEABLE(pre_bv, &bvec)) {
bv_len[i - 1] += cur_bv_len;
goto out_bv_merged;
}
usr_buf[i] = buffer;
bv_len[i] = cur_bv_len;
if (i < TEST_DMA_BUF_MAX_NUM) { // !!!!!!
kvaddr[i] = pci_alloc_consistent(io_que->ssd_dev->pci_dev, COHERENT_DMA_BUF_SZ, &dma_addr[i]); // !!!
if (NULL == kvaddr[i]) {
printk("pci_alloc_consistent kvaddr[%u] failed\n", i);
goto out_pci_alloc_consistent;
}
}
i++;
out_bv_merged:
__bio_kunmap_atomic(buffer);
pre_bv = &bvec;
isnt_first_bio_vec = 1u;
}
upper_segs = __MIN(upper_segs, i); // !!!!!!
desc_kvbar = pci_alloc_consistent(io_que->ssd_dev->pci_dev, /*2*phys_segs*TRANS_DESC_ALIGN*/COHERENT_DMA_BUF_SZ, &desc_dma_bar); // !!!
if (NULL == desc_kvbar) {
printk("pci_alloc_consistent desc_kvbar failed\n");
goto out_pci_alloc_consistent_desc_kvbar;
}
for (k = 0; k < upper_segs; k++) {
//writel(dma_addr[k], (uint64_t *)io_que->ssd_dev->pci_bar + k); // !!!!!!64 bits, cannot use writel
writel((uint32_t)(dma_addr[k] >> 32), (uint32_t *)io_que->ssd_dev->pci_bar + (k << 1)); // !!!address of high 32 bits is lower
writel((uint32_t)dma_addr[k], (uint32_t *)io_que->ssd_dev->pci_bar + ((k << 1) | 1));
}
if (dir == WRITE) {
for (h = 0; h < upper_segs; h++)
memcpy(kvaddr[h], usr_buf[h], bv_len[h]);
for (k = 0; k < upper_segs; k++) {
fill_trans_desc(desc_kvbar, 2*k, 0,
TRANS_BRAM_BASE_ADDR + k*sizeof(uint64_t),
TRANS_BRAM_BASE_ADDR + AXI_PCIE_CTL_OFFSET + AXIBAR2PCIEBAR_1U_OFFSET,
sizeof(uint64_t));
fill_trans_desc(desc_kvbar, 2*k + 1, (k == upper_segs - 1) ? 1 : 0,
AXI_BAR1, DDR3_BASE_ADDR + byte_offset, bv_len[k]);
byte_offset += bv_len[k];
}
writel((unsigned long)desc_dma_bar,
(unsigned char *)io_que->ssd_dev->pci_bar + AXI_PCIE_CTL_OFFSET + AXIBAR2PCIEBAR_2L_OFFSET); // !!!desc
writel(AXI_BAR2,
(unsigned char *)io_que->ssd_dev->pci_bar + AXI_CDMA_LITE_OFFSET + CURDESC_PNTR_OFFSET);
writel(AXI_BAR2 + (2*upper_segs - 1)*TRANS_DESC_ALIGN,
(unsigned char *)io_que->ssd_dev->pci_bar + AXI_CDMA_LITE_OFFSET + TAILDESC_PNTR_OFFSET); // !!!last step, start sg
if (cdma_wait_idle(io_que->ssd_dev->pci_bar)) {
printk("couldn't wait to idle2\n");
for (h = 0; h < upper_segs; h++)
print_desc_stat(desc_kvbar, h);
}
} else { // READ
for (k = 0; k < upper_segs; k++) {
fill_trans_desc(desc_kvbar, 2*k, 0,
TRANS_BRAM_BASE_ADDR + k*sizeof(uint64_t),
TRANS_BRAM_BASE_ADDR + AXI_PCIE_CTL_OFFSET + AXIBAR2PCIEBAR_1U_OFFSET,
sizeof(uint64_t));
fill_trans_desc(desc_kvbar, 2*k + 1, (k == upper_segs - 1) ? 1 : 0,
DDR3_BASE_ADDR + byte_offset, AXI_BAR1, bv_len[k]);
byte_offset += bv_len[k];
}
writel((unsigned long)desc_dma_bar,
(unsigned char *)io_que->ssd_dev->pci_bar + AXI_PCIE_CTL_OFFSET + AXIBAR2PCIEBAR_2L_OFFSET); // !!!desc
writel(AXI_BAR2,
(unsigned char *)io_que->ssd_dev->pci_bar + AXI_CDMA_LITE_OFFSET + CURDESC_PNTR_OFFSET);
writel(AXI_BAR2 + (2*upper_segs - 1)*TRANS_DESC_ALIGN,
(unsigned char *)io_que->ssd_dev->pci_bar + AXI_CDMA_LITE_OFFSET + TAILDESC_PNTR_OFFSET); // last step
if (cdma_wait_idle(io_que->ssd_dev->pci_bar)) {
printk("couldn't wait to idle2\n");
for (h = 0; h < upper_segs; h++)
print_desc_stat(desc_kvbar, h);
}
for (h = 0; h < upper_segs; h++)
memcpy(usr_buf[h], kvaddr[h], bv_len[h]); // pay attention to SA and DA!!!
}
pci_free_consistent(io_que->ssd_dev->pci_dev, /*(2*phys_segs)*TRANS_DESC_ALIGN*/COHERENT_DMA_BUF_SZ, desc_kvbar, desc_dma_bar);
out_pci_alloc_consistent_desc_kvbar:
out_pci_alloc_consistent:
for (j = (i >= TEST_DMA_BUF_MAX_NUM ? TEST_DMA_BUF_MAX_NUM-1 : i-1); j >= 0; j--)
pci_free_consistent(io_que->ssd_dev->pci_dev, COHERENT_DMA_BUF_SZ, kvaddr[j], dma_addr[j]); // j!!!
kfree(usr_buf);
out_kmalloc_usr_buf:
kfree(bv_len);
out_kmalloc_bv_len:
kfree(dma_addr);
out_kmalloc_dma_addr:
kfree(kvaddr);
out_kmalloc_kvaddr:
bio_endio(bio); // !!!
}
/*
static void free_scatter_map(struct ssd_dev *ssd_dev, struct io_cmd *io_cmd)
{
dma_unmap_sg(&ssd_dev->pci_dev->dev, io_cmd->scatlist, io_cmd->len,
bio_data_dir(io_cmd->bio) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE); // !!!
dma_free_coherent(&ssd_dev->pci_dev->dev, PAGE_SIZE, io_cmd->kvaddr,
io_cmd->dma_addr);
kfree(io_cmd->scatlist);
}
*/
/*
static void test_bio_complete(struct ssd_dev *ssd_dev, struct io_que *dev_que) // !!!???logic wrong?
{
struct bio *bio;
struct io_cmd *io_cmd;
io_cmd = dev_que->io_cmd;
free_scatter_map(ssd_dev, io_cmd); // above
bio = io_cmd->bio; // !!!has been saved before
if (bio->bi_vcnt == bio->bi_iter.bi_idx)
bio_endio(bio); // !!!
dev_que->is_busy = 0; // !!!not busy
if (bio_list_peek(&dev_que->bio_lst))
wake_up_process(dev_que->task_s);
}
*/
// !!!consumer: has been binded below
static int test_kthread_fn(void *data)
{
struct io_que *dev_que;
struct bio *bio;
dev_que = (struct io_que *)data;
if (dev_que == NULL)
printk("in test_kthread_fn dev_que is null!\n");
do {
//struct bio *bio;
//if (dev_que->is_busy) // !!!!!!DMA channel is busy
//goto sleep_this_thread;
if (bio_list_peek(&dev_que->bio_lst)) {
spin_lock(&dev_que->lock);
bio = bio_list_pop(&dev_que->bio_lst); // !!!!!!get bio
spin_unlock(&dev_que->lock);
//printk("test_kthread_fn: get a bio\n");
/*if (make_bio_request(dev_que, bio)) { // above
spin_lock(&dev_que->lock);
bio_list_add_head(&dev_que->bio_lst, bio); // add from head
spin_unlock(&dev_que->lock);
}*/
test_process_bio(dev_que, bio); // !!!!!!
}
//test_bio_complete(ssd_dev, dev_que); // above, orginally it is here!!! why it is not defined but can be compiled??????
//sleep_this_thread:
schedule(); // !!!make this thread sleep!!!!necessary!or NMI watch dog error
} while (!kthread_should_stop()); // !!!kthread.c
return 0;
}
/*
static irqreturn_t irq_handler(int irq, void *dev_id) // !!!
{
struct ssd_dev *ssd_dev;
struct io_que *dev_que;
printk("irq_handler\n");
dev_que = (struct io_que *)dev_id; // !!!
ssd_dev = dev_que->ssd_dev; // !!!
//spin_lock_irq(&dev_que->lock);
test_bio_complete(ssd_dev, dev_que); // !!!above
//spin_unlock_irq(&dev_que->lock);
return IRQ_HANDLED;
}
*/
static int alloc_kthread_ret;
static int alloc_kthread(struct io_que *dev_que) // !!!create consumer and make it run
{
dev_que->task_s = kthread_run(&test_kthread_fn, dev_que, KTHREAD_NAME); // !!!kthread.h kthread.c
if (IS_ERR(dev_que->task_s)) { /* def in <linux/err.h> */
printk("err: kthread_run\n");
return PTR_ERR(dev_que->task_s);
} else
return 0;
}
static void free_kthread(struct io_que *dev_que)
{
if (kthread_stop(dev_que->task_s) == 0) // kthread.c, struct task_struct *
printk("threadfn has returned\n");
}
// !!!producer: binded with make_request below, only to add bio to the bio_list. blk_qc_t is unsigned int
static void test_make_request_fn(struct request_queue *queue, struct bio *bio)
{
struct io_que *dev_que;
dev_que = (struct io_que *)queue->queuedata; // !!!
spin_lock(&dev_que->lock);
bio_list_add(&dev_que->bio_lst, bio); // !!!add from tail
spin_unlock(&dev_que->lock);
//printk("test_make_request_fn: add a bio\n");
}
static struct io_que *alloc_io_que_ret;
// !!!!!!ssd_dev already alloc, and it's disk already alloc.
static struct io_que *alloc_io_que(struct ssd_dev *ssd_dev)
{
struct io_que *dev_que; // const
dev_que = (struct io_que *)kmalloc(sizeof(struct io_que), GFP_KERNEL); // !!!
if (dev_que == NULL) {
printk("err_alloc_dev_que\n");
goto err_alloc_dev_que;
}
ssd_dev->dev_que = dev_que; // !!!!!!
dev_que->ssd_dev = ssd_dev; // !!!!!!
spin_lock_init(&dev_que->lock); // only for init
bio_list_init(&dev_que->bio_lst); // only for init, struct bio_list, bl->head = bl->tail = NULL; comes before consumer!!!
dev_que->is_busy = 0; // !!!only for init
dev_que->io_cmd = (struct io_cmd *)kmalloc(sizeof(struct io_cmd), GFP_KERNEL); // !!!!!!
if (dev_que->io_cmd == NULL) {
printk("err_alloc_io_cmd\n");
goto err_alloc_io_cmd;
}
/*
dev_que->io_cmd->kvaddr = dma_alloc_coherent(&dev_que->ssd_dev->pci_dev->dev, COHERENT_DMA_BUF_SZ,
&dev_que->io_cmd->dma_addr, GFP_ATOMIC | GFP_DMA); // !!!!!!4MB
if (dev_que->io_cmd->kvaddr == NULL) {
printk("in alloc_io_que: err_dma_pool_alloc\n");
goto err_dma_alloc_coherent;
}
writel((unsigned long)dev_que->io_cmd->dma_addr,
(unsigned char *)dev_que->ssd_dev->pci_bar + AXIBAR2PCIEBAR1_OFFSET_L); // !!!!!!map dma_addr(fixed position) to AXI_BAR
writel((unsigned long)(dev_que->io_cmd->dma_addr >> 32),
(unsigned char *)dev_que->ssd_dev->pci_bar + AXIBAR2PCIEBAR1_OFFSET_U);
printk("before trans stat_reg: %x\n", readl((unsigned char *)dev_que->ssd_dev->pci_bar + C_BASEADDR + CDMA_STAT_REG_OFFSET));
*/
alloc_kthread_ret = alloc_kthread(dev_que);
if (alloc_kthread_ret) { // !!!!!!consumer comes before producer
printk("err: alloc_kthread\n");
goto err_alloc_kthread;
}
dev_que->ssd_dev->disk->queue = blk_alloc_queue(GFP_KERNEL); // !!!!!!
if (dev_que->ssd_dev->disk->queue == NULL) {
printk("err: blk_alloc_queue\n");
goto err_blk_alloc_queue;
}
dev_que->ssd_dev->disk->queue->queuedata = dev_que; // !!!void *queuedata, point to itself
dev_que->ssd_dev->disk->queue->queue_flags = QUEUE_FLAG_DEFAULT; // it is needed
//queue_flag_set(QUEUE_FLAG_NOMERGES, dev_que->ssd_dev->disk->queue); /* disable merge attempts */
queue_flag_set(QUEUE_FLAG_NONROT, dev_que->ssd_dev->disk->queue); /* non-rotational device (SSD) */
blk_queue_make_request(dev_que->ssd_dev->disk->queue, &test_make_request_fn); // !!!binded make_request_fn(producer) to the queue
return dev_que;
err_blk_alloc_queue:
free_kthread(dev_que);
err_alloc_kthread:
//dma_free_coherent(&dev_que->ssd_dev->pci_dev->dev, COHERENT_DMA_BUF_SZ, dev_que->io_cmd->kvaddr, dev_que->io_cmd->dma_addr); // !!!
//err_dma_alloc_coherent:
kfree(dev_que->io_cmd);
err_alloc_io_cmd:
kfree(dev_que);
err_alloc_dev_que:
return NULL;
}
static void free_io_que(struct ssd_dev *ssd_dev, struct io_que *dev_que)
{
blk_cleanup_queue(dev_que->ssd_dev->disk->queue);
free_kthread(dev_que);
//dma_free_coherent(&dev_que->ssd_dev->pci_dev->dev, COHERENT_DMA_BUF_SZ, dev_que->io_cmd->kvaddr, dev_que->io_cmd->dma_addr); // !!!
kfree(dev_que->io_cmd);
kfree(dev_que);
}
static int test_ssd_open(struct block_device *bdev, fmode_t mode)
{
//printk("test_ssd_open\n");
return 0;
}
static void test_ssd_release(struct gendisk *disk, fmode_t mode)
{
//printk("test_ssd_release\n");
}
static struct block_device_operations ssd_fops = {
.open = &test_ssd_open,
.release = &test_ssd_release,
.owner = THIS_MODULE,
};
static int blkdev_major, test_ssd_init_ret;
static int test_ssd_init(struct ssd_dev *ssd_dev) // !!!
{
struct io_que *dev_que;
int result = -ENOMEM;
printk("blkdev init begin\n");
blkdev_major = register_blkdev(0, TEST_SSD_DEV_NAME); // !!!try to allocate any unused major number.
if (blkdev_major < 0) {
printk("err: register_blkdev\n");
goto err_register_blkdev;
}
ssd_dev->disk = alloc_disk(TEST_SSD_PARTITONS); // !!!
if (ssd_dev->disk == NULL) {
printk("err: alloc_disk\n");
result = -ENOMEM;
goto err_alloc_disk;
}
ssd_dev->disk->major = blkdev_major;
ssd_dev->disk->first_minor = 0; // !!!!!!
ssd_dev->disk->minors = TEST_SSD_MINORS;
sprintf(ssd_dev->disk->disk_name, "%s" , TEST_SSD_DEV_NAME); // !!!
ssd_dev->disk->fops = &ssd_fops;
ssd_dev->disk->private_data = ssd_dev; // !!!
//ssd_dev->disk->driverfs_dev = &ssd_dev->pci_dev->dev; // genhd.h: struct device *driverfs_dev; // FIXME: remove
set_capacity(ssd_dev->disk, TEST_DEV_MEM_SZ >> 9); // in unit of sector(512 bytes long independently)
alloc_io_que_ret = alloc_io_que(ssd_dev); // !!!above, set ssd_dev->disk->queue
dev_que = alloc_io_que_ret;
if (dev_que == NULL) {
printk("err: alloc_io_que\n");
result = -ENOMEM;
goto err_alloc_io_que;
}
add_disk(ssd_dev->disk); // !!!!!!add partitioning information to kernel list
// "ssd_dev->pci_dev->irq" init in pci_enable_msi func. dev_que is for param of isr
//if (request_irq(ssd_dev->pci_dev->irq, &irq_handler,
//IRQF_NOBALANCING | IRQF_SHARED, ssd_dev->disk->disk_name, dev_que) < 0) {
//printk("err_request_irq\n");
//goto err_request_irq;
//}
printk("blkdev init end\n");
return 0;
//err_request_irq:
err_alloc_io_que:
del_gendisk(ssd_dev->disk); // !!!
//put_disk(ssd_dev->disk);
err_alloc_disk:
unregister_blkdev(blkdev_major, TEST_SSD_DEV_NAME); // !!!
err_register_blkdev:
return result;
}
static int test_ssd_remove(struct ssd_dev *ssd_dev)
{
struct io_que *dev_que;
printk("test_ssd_remove begin\n");
dev_que = ssd_dev->dev_que;
//free_irq(ssd_dev->pci_dev->irq, dev_que);
if (alloc_io_que_ret)
free_io_que(ssd_dev, dev_que);
del_gendisk(ssd_dev->disk);
//put_disk(ssd_dev->disk);
unregister_blkdev(blkdev_major, TEST_SSD_DEV_NAME); // !!!
printk("test_ssd_remove end\n");
return 0;
}
static struct pci_device_id test_id_tbl[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_XILINX, TEST_PCI_DEVICE_ID_XILINX), },
{0,},
};
MODULE_DEVICE_TABLE(pci, test_id_tbl);
/*
static void cdma_init_simple(void __iomem *pci_bar)
{
unsigned long read_val;
// default simple mode
read_val = readl((unsigned char *)pci_bar + AXI_CDMA_LITE_OFFSET + CDMACR_OFFSET);
if ((read_val >> 3) & 0x1)
printk("it's sg mode\n");
else
printk("it's simple mode\n");
if ((read_val >> 2) & 0x1)
printk("reset in progress\n");
else
printk("normal operation\n");
read_val = readl((unsigned char *)pci_bar + AXI_CDMA_LITE_OFFSET + CDMASR_OFFSET);
if ((read_val >> 1) & 0x1)
printk("cdma is idle\n");
else
printk("cdma is not idle\n");
}
*/
static uint32_t bars;
static int test_probe_ret;
static int test_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) // !!!
{
struct ssd_dev *ssd_dev;
resource_size_t res_start, res_len; // actually it's integer type
int result = -ENOMEM;
printk("pci_driver_probe begin with vendor=%x, device=%x\n", id->vendor, id->device);
ssd_dev = (struct ssd_dev *)kmalloc(sizeof(struct ssd_dev), GFP_KERNEL); // !!!!!!
if (ssd_dev == NULL) {
printk("err: kmalloc ssd_dev\n");
goto err_kmalloc_ssd_dev;
}
ssd_dev->pci_dev = pci_dev; // !!!!!!
if (pci_enable_device(pci_dev) < 0) { // !!!
printk("err: pci_enable_device\n");
goto err_pci_enable_device;
}
pci_set_master(pci_dev); // !!!enables bus-mastering for device dev
//bars = pci_select_bars(pci_dev, IORESOURCE_MEM); // !!!<linux/ioport.h>
bars = 0; // !!!!!!it's already set in the hardware project
if (pci_request_selected_regions(pci_dev, bars, TEST_PCIE_DEV_NAME)) { // actually using __request_mem_region
printk("err: pci_request_selected_regions\n");
goto err_pci_request_selected_regions;
}
res_start = pci_resource_start(pci_dev, bars);
res_len = pci_resource_len(pci_dev, bars);
printk("pci_res_start=%lu, pci_res_len=%lu\n", (unsigned long)res_start, (unsigned long)res_len);
//request_mem_region(pci_resource_start(pci_dev, bars), pci_resource_len(pci_dev, bars), TEST_PCIE_DEV_NAME);
/* !!!associate with drivers/pci/msi.c, using pci_enable_msi_range,
* updates the @dev's irq member to the lowest new interrupt number;
*/
//if (pci_enable_msi(pci_dev) < 0)
//printk("pci_enable_msi: an error occurs\n");
//ssd_dev->irq = pci_dev->irq; // !!!!!!
pci_set_drvdata(pci_dev, ssd_dev); // !!!bind ssd_dev to pci_dev, for later use
//if (pci_set_dma_mask(pci_dev, DMA_BIT_MASK(64)) < 0) // if return err
//printk("err: pci_set_dma_mask\n");
//if (pci_set_consistent_dma_mask(pci_dev, DMA_BIT_MASK(64)) < 0) // both needed
//printk("pci_set_consistent_dma_mask err\n");
// !!!!!!it's bars, not 0? set here! __iomem is needed
ssd_dev->pci_bar = ioremap(res_start, res_len); // !!!
if (ssd_dev->pci_bar == NULL) {
printk("err: ioremap\n");
goto err_ioremap;
}
//writel(0x0, (unsigned char *)ssd_dev->pci_bar + 0xc000); // !!!!!!CDMA CTL 设定为single模式
//if ((readl((unsigned char *)ssd_dev->pci_bar + 0xc000) >> 3) & 0x1)
//printk("CDMA CTL settings err: is not single mode\n");
//printk("CDMA STATUS is %x\n", readl((unsigned char *)ssd_dev->pci_bar + 0xc004)); // !!!
//cdma_init_simple(ssd_dev->pci_bar);
cdma_set_sg_mode(ssd_dev->pci_bar); // !!!
printk("pci bus init has successfully ended\n");
test_ssd_init_ret = test_ssd_init(ssd_dev);
if (test_ssd_init_ret) { // above
printk("err_test_ssd_init\n");
goto err_ssd_init;
}
printk("pci_driver_probe end\n");
test_probe_ret = 0;
return 0;
err_ssd_init:
iounmap(ssd_dev->pci_bar);
err_ioremap:
pci_set_drvdata(pci_dev, NULL); // !!!where should it be?
//pci_disable_msi(pci_dev);
pci_release_selected_regions(pci_dev, bars); // !!!
err_pci_request_selected_regions:
pci_clear_master(pci_dev); // !!!
pci_disable_device(pci_dev);
err_pci_enable_device:
kfree(ssd_dev);
err_kmalloc_ssd_dev:
test_probe_ret = result;
return result;
}
static void test_remove(struct pci_dev *pci_dev)
{
struct ssd_dev *ssd_dev;
printk("pci_driver_remove begin\n");
if (test_probe_ret == 0) {
ssd_dev = (struct ssd_dev *)pci_get_drvdata(pci_dev); // has been binded before
if (test_ssd_init_ret == 0)
test_ssd_remove(ssd_dev); // above
iounmap(ssd_dev->pci_bar); // !!!
pci_set_drvdata(pci_dev, NULL);
//pci_disable_msi(pci_dev);
pci_release_selected_regions(pci_dev, bars); // !!!original: pci_release_regions(pci_dev);
pci_clear_master(pci_dev);
pci_disable_device(pci_dev);
kfree(ssd_dev);
}
printk("pci_driver_remove end\n");
}
static struct pci_driver pci_driver_inst = {
.name = TEST_PCIE_DEV_NAME,
.id_table = test_id_tbl,
.probe = &test_probe,
.remove = &test_remove,
};
static int __init test_module_init(void)
{
int result = -EBUSY;
printk("module_init_fn begin\n");
if (pci_register_driver(&pci_driver_inst)) {
printk("err_register_driver\n");
goto err_register_driver;
}
printk("module_init_fn end\n");
return 0;
err_register_driver:
return result;
}
module_init(test_module_init);
static void __exit test_module_exit(void)
{
printk("module_exit_fn begin\n");
pci_unregister_driver(&pci_driver_inst);
printk("module_exit_fn end\n");
}
module_exit(test_module_exit);
MODULE_LICENSE("GPL"); // !!!
更多推荐
PCIe-块设备驱动-SG DMA
发布评论