PCIE开发流程讲解学习

PCIE开发流程

PCIE开发流程

前言：对于USB、PCIE设备这种挂接在总线上的设备而言，USB、PCI只是它们的”工作单位”，它们需要向”工作单位”注册（使用usb_driver，pci_driver），并接收”工作单位”的管理（被调入probe()、调出disconnect/remove()、放假suspend()/shutdown()、继续上班resume()等），但设备本身可能是一个工程师、一个前台或者一个经理，因此做好工程师，前台或者经理是其主题工作，这部分对应于字符设备驱动，tty设备驱动，网络设备驱动等。

第一节 整体构成

整个驱动程序的开发应该包括三个大的部分

1.1 驱动模块的加载与卸载

xxx_init_module()注册pci_driver设备。

xxx_cleanup_module()注销pci_driver设备。

1.2 pci_driver成员函数的初始化

xxx_probe()完成PCI设备初始化，注册字符设备

xxx_remove()完成PCI设备释放，注销字符设备

1.3 字符设备file_operations成员函数

用于实现上层应用程序对下层驱动程序调用时的调用函数。

xxx_open()

xxx_release()

xxx_ioctl()

xxx_read()

xxx_write()

第二节 PCIE设备实现细节

由于PCIE设备的驱动的开发都是按照一个统一规范的框架进行的。因此以一个字符设备为例说明这个框架的实现机制。在所有PCIE驱动开发的过程中

2.1 驱动程序的初始化和注销

涉及的函数为module_init(xxx_init_module)，并在init中完成的功能为注册PCIE设备，具体函数内容如下所示：

注销涉及的函数为module_exit(xxx_cleanup_module)在exit中完成的功能为注销PCIE设备，具体函数内容如下所示：

2.2 PCIE设备的注册

在模块的初始化过程中，首先是注册PCIE设备，使用函数为pci_register_driver(&xxx_pci_driver)，输入变量指明了PCIE结构体，如下所示：

#define XXX_MODULE_NAME \"xxx_audio\"

static struct pci_driver xxx_pci_driver = {

.name = XXX_MODULE_NAME,

.id_table = xxx_pci_tbl,

.probe = xxx_probe,

.remove = __devexit_p(xxx_remove),

#ifdef CONFIG_PM

.suspend = xxx_pm_suspend,

.resume = xxx_pm_resume,

#endif /* CONFIG_PM */

};

结构体中name指明PCIE模块的名称，id_table指明了PCIE的设备驱动号也就是为哪个设备进行驱动等。

其中probe函数完成PCI设备的初始化以及其设备本身身份（字符，TTY，网络等）的驱动注册。也是驱动注册中最重要的函数。

probe函数讲解

1、 首先使能pci设备，pci_enable_device(pci_dev)，该函数主要作用是调用底

层代码初始化PCI配置空间命令寄存器的I/O位和memory位。

2、 设置成总线主DMA模式，pci_set_dma_mask(pci_dev, XXX_DMA_MASK)

用于实现对dma设备的设置。

3、 读取PCI的配置信息，使用的函数是pci_resource_start (pci_dev, 1)获取设

备的内存基地址和所有BAR的长度，

4、 调用ioremap完成配置信息的映射，可以配置PCI定义的寄存器BAR设置

的空间用于映射DMA的寄存器。

4、 申请I/O资源，request_region(card->ac97base, 256,

card_names[pci_id->driver_data])

5、 注册字符/网络设备涉及到的函数为

cdev_init(xxx_cdev,&xxx_fops);/*注册驱动*/

register_chrdev_region(xxx_dev_no,1,XXX);/*申请设备号*/

cdev_add(xxx_cdev);/*添加字符设备*/

request_irq(card->irq, &xxx_interrupt, SA_SHIRQ, card_names[pci_id-

>driver_data], card))/*申请中断以及注册中断处理程序*/

remove函数讲解

1、 释放I/O资源 pci_release_regions(pdev)

2、 禁止PCI设备 pci_disable_device(pdev)

释放占用的设备号register_chrdev_region(xxx_dev_no,1,XXX);

3、 注销字符设备cdev_del(&xxx_dev.cdev)。

2.3 设备的file_operations操作

在probe中需要注册字符设备，实现应用程序对PCIE设备的调用，例如打开，读取，控制等。这些功能都是通过file_operations操作接口实现。例如用户使用该设备完成读取操作，那么用户的过程为open(),read()。而用户调用的这些函数对于linux来说这些调用都会变成系统调用，并指向改设备对应的open(),read()函数，对于该设备来说，指向了xxx_open和xxx_read。

static struct file_operations xxx_fops = {

.owner = THIS_MODULE,

.llseek = no_llseek,

.read = sgma_read,

.write = xxx_write,

.poll = xxx_poll,

.ioctl = xxx_ioctl,

.mmap = xxx_mmap,

.open = xxx_open,

.release = xxx_release,

};

接下来，需要实现上面的这些操作，然后就能实现用户对设备的调用。

static int i810_open(struct inode *inode, struct file *file)

{

^^^^^^^^

}

2.4 其他说明

a)、中断

在PCIE中可以使用request_irq共享中断或者pci_enable_msi消息告知申请中断，不同之处在于前者在扫描PCI的时候自动为设备分配好中断号，这样存在多个设备共享中断号的情况。MSI中断是在调用初始化函数pci_enable_msi()才分配中断号，可以保证设备的中断号不会与其他设备共用，从而避免了中断共享能够提高整体性能，但是MSI中断的使用需要Linux操作系统特殊的支持，不具有普遍的适用性。传统的中断，由于资源号是有限的，常常涉及到多个设备共享同一个中断号，在中断的处理过程中要依次调用每个中断处理函数来判断中断是不是目标设备发出，这会消耗系统性能。

第三节 示例程序

#include \"card.h\"

#include

#include

#define DMA_MASK 0xffffffff

#define test_dri_major 249 // 主设备号

//#define INT_ASSERT_W 0x02 // DMA Write Complete

//#define INT_ASSERT_R 0x10 // DMA Read Complete

/* PCI 驱动基本框架 ，为下面的设备进行驱动*/

static struct pci_device_id card_ids[] = {

{PCI_DEVICE(PCI_VENDOR_ID_XILINX,PCI_DEVICE_ID_EP_PIPE),},

{0,}

};

MODULE_DEVICE_TABLE(pci,card_ids);

/*probe和remove基本函数*/

static int card_probe(struct pci_dev *pci_dev, const struct pci_device_id *id);

static void card_remove(struct pci_dev *pdev);

/*pci_driver 结构体*/

static struct pci_driver card_driver = {

.name = DEV_NAME,

.id_table = card_ids,

.probe = card_probe,

.remove = card_remove,

};

static int __init card_init(void)

{

int result;

result = pci_register_driver(&card_driver);

return result;

}

static void __exit card_exit(void)

{

pci_unregister_driver(&card_driver);

}

module_init(card_init);

module_exit(card_exit);

/* PCI 驱动基本框架 */

/* 特定设备私有数据结构 */

struct card_private {

struct pci_dev* pci_dev;

void* pci_bar0;

//wait_queue_head_t * dma_write_wait;

//wait_queue_head_t * dma_read_wait;

};

/* 特定设备私有数据结构 */

static struct card_private *adapter;

//static DECLARE_WAIT_QUEUE_HEAD(dma_write_wait);

//static int flag = 1;

// 将文件操作与分配的设备号相连

static const struct file_operations card_fops =

{

.owner = THIS_MODULE,

//.ioctl = card_ioctl,

.open = card_open,

.release= card_release,

.read = card_read,

.write = card_write,

};

static int card_probe(struct pci_dev *pdev, const struct pci_device_id *id)

{

unsigned long phymem;

void __iomem *mem;

u_int8_t csz;

u32 val;

int result;

/*配置PCI设备*/

if (pci_enable_device(pdev))

return -EIO;

/* XXX 32-bit addressing only */

if (pci_set_dma_mask(pdev, 0xffffffff)) {

printk(KERN_ERR \"ath_pci: 32-bit DMA not available\\n\");

goto bad;

}

//pci_write_config_word(pdev, 0x04, 0x0007);

/*配置PCI寄存器，首先调用pci_read_config_byte进行读取PCI配置空间，并将值返回给csz

* Cache line size is used to size and align various

* structures used to communicate with the hardware.

*/

pci_read_config_byte(pdev, PCI_CACHE_LINE_SIZE, &csz);

if (csz == 0) {

/*

* Linux 2.4.18 (at least) writes the cache line size

* register as a 16-bit wide register which is wrong.

* We must have this setup properly for rx buffer

* DMA to work so force a reasonable value here if it

* comes up zero.

*/

csz = L1_CACHE_BYTES / sizeof(u_int32_t);

pci_write_config_byte(pdev, PCI_CACHE_LINE_SIZE, csz);

}

/*

* The default setting of latency timer yields poor results,

* set it to the value used by other systems. It may be worth

* tweaking this setting more.

*配置PCI配置空间

*/

pci_write_config_byte(pdev, PCI_LATENCY_TIMER, 0xa8);

/*设置成总线主模式*/

pci_set_master(pdev);

/*读取寄存器信息

* Disable the RETRY_TIMEOUT register (0x41) to keep

* PCI Tx retries from interfering with C3 CPU state.

*

* Code taken from ipw2100 driver - jg

*/

pci_read_config_dword(pdev, 0x40, &val);

if ((val & 0x0000ff00) != 0)

pci_write_config_dword(pdev, 0x40, val & 0xffff00ff);

// 获得BAR0空间的基地址，该地址为存储器域的物理地址;

phymem = pci_resource_start(pdev, 0);

if (!request_mem_region(phymem, pci_resource_len(pdev, 0), DEV_NAME)) {

printk(KERN_ERR \"card_driver: cannot reserve PCI memory region\\n\");

goto bad;

}

// 将存储器域的物理地址映射为虚拟地址;

mem = ioremap(phymem, pci_resource_len(pdev, 0));

if (!mem) {

printk(KERN_ERR \"card_driver: cannot remap PCI memory region\\n\") ;

goto bad1;

}

adapter = kmalloc(sizeof(struct card_private), GFP_KERNEL);

if (unlikely(!adapter)){

return -ENOMEM;

}

adapter -> pci_dev = pdev;

adapter -> pci_bar0 = mem;

// 注册设备驱动程序

result = register_chrdev(test_dri_major, DEV_NAME, &card_fops);

if (unlikely(result)){

printk(KERN_ERR \"card_driver: no memory for device state\\n\");

goto bad2;

}

/*

//init_waitqueue_head(adapter->dma_write_wait);

//init_waitqueue_head(adapter->dma_read_wait);

result = pci_enable_msi(pdev);

if (unlikely(result)){

//PDEBUG(\"cannot enable msi ... \\n\");

goto bad3;

}

result = request_irq(pdev -> irq, card_interrupt, 0, DEV_NAME, NULL);

if (unlikely(result)){

//PDEBUG(\"request interrupt failed ... \\n\");

goto bad3;

printk(KERN_DEBUG \"request_irq(pdev -> irq, card_interrupt, }

*/

return 0;

//bad3:

// unregister_chrdev(test_dri_major, DEV_NAME);

0, DEV_NAME, NULL);\");

bad2:

iounmap(mem);

bad1:

release_mem_region(phymem, pci_resource_len(pdev, 0));

bad:

pci_disable_device(pdev);

return (-ENODEV);

}

static void card_remove(struct pci_dev *pdev)

{

//pci_disable_msi(pdev);

//if(pdev->irq)

// free_irq(pdev->irq, pdev);

iounmap(adapter -> pci_bar0);

release_mem_region(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0));

pci_disable_device(pdev);

unregister_chrdev(test_dri_major, DEV_NAME);

}

//打开设备文件系统调用对应的操作

static int card_open(struct inode *inode, struct file *filp)

{

return 0;

}

//关闭设备文件系统调用对应的操作

static int card_release(struct inode *inode, struct file *filp)

{

return 0;

}

//读取设备信息

static ssize_t card_read(struct file *file, char __user *buf, loff_t *f_pos)

{

void* virt_addr = NULL;

dma_addr_t dma_write_addr;

u32 base, w_ddr2, w_addr,w_size,cst_32,w_counter,dma_cst;

u32 ddr2;

int i;

i = 0;

/*读取寄存器的值，0x10的位置也就是BAR的初始地址*/

pci_read_config_dword(adapter->pci_dev, 0x10, &base);

size_t count,

printk(KERN_DEBUG \" pci_read_config_dword, base:%x\\n\

printk(KERN_DEBUG \"adater -> pci_bar0 %lx\\n\-> pci_bar0));

/*********Request virt_addr(kernel) for Read(DMA write)********/

virt_addr = kmalloc(count, GFP_KERNEL|__GFP_DMA);

if(unlikely(!virt_addr))

{

//PDEBUG(\"cannot alloc rx memory you want ... \\n\");

return -EIO;

}

printk(KERN_DEBUG \" virt_addr(kernel):%x\\n\

/*********Request virt_addr(kernel) for Read(DMA write)********/

/**********************dma_write_addr************************/

//将存储器域的虚拟地址virt_addr转化为pci总线域的物理地址dma_write_addr,

供card的DMA控制器使用。

dma_write_addr = pci_map_single(adapter->pci_dev, virt_addr, count, PCI_DMA_FROMDEVICE);

if(unlikely(pci_dma_mapping_error(adapter->pci_dev, dma_write_addr)))

{

//PDEBUG(\"RX DMA MAPPING FAIL...\\n\");

goto err_kmalloc;

}

printk(KERN_DEBUG \" dma_write_addr:%x\\n\

/**********************dma_write_addr************************/

/**********************BAR0 kong jian ***********************/

// START, w_counter读取BAR0空间

w_counter = ioread32((adapter ->

pci_bar0+WRITE_DMA_COUNTER_OFFSET));

printk(KERN_DEBUG \" START, w_counter: %x\

// w_ddr2 DMA写的原地址

iowrite32(*f_pos,(adapter -> pci_bar0 + WRITE_DDR2_SA_OFFSET));

ddr2 = ioread32((adapter -> pci_bar0 + WRITE_DDR2_SA_OFFSET));

printk(KERN_DEBUG \" WRITE_DDR2_SA_OFFSET: %x\

// w_addr

// Lower 32-bit address of system memory buffer for DMA operation.

iowrite32(dma_write_addr,(adapter -> pci_bar0 WRITE_HOST_DA_L_OFFSET));

// w_size

// Write DMA TLP Size Register(from DDR2 to mm).

iowrite32(count,(adapter -> pci_bar0 + WRITE_SIZE_OFFSET));

// Write DMA Control && Status Register

write +

dma_cst = 0;

dma_cst = ioread32((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst1: %x\

dma_cst = dma_cst | 0x1;

printk(KERN_DEBUG \"dma_cst2: %x\

// dma_cst

//writel(dma_cst,(void __iomem *) (unsigned long) (adapter -> pci_bar0 + DMA_CST_OFFSET));

iowrite32(dma_cst,(adapter -> pci_bar0 + DMA_CST_OFFSET));

dma_cst = ioread32((adapter -> pci_bar0+DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst3: %x\

while( (!(dma_cst & 0x2))&&(i<1000) )

{

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

i++;

printk(KERN_DEBUG

\"#########################################\");

}

i = 0;

dma_cst = dma_cst | 0x2;

iowrite32(dma_cst,(adapter -> pci_bar0 + DMA_CST_OFFSET));

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst4: %x\

//interruptible_sleep_on((adapter->dma_write_wait));

//wait_event_interruptible(dma_write_wait,flag);

//flag = 0;

//test (w_ddr2, w_addr,w_size,cst_32,w_counter)

w_ddr2 = ioread32((adapter -> pci_bar0 + WRITE_DDR2_SA_OFFSET));

w_addr = ioread32((adapter -> pci_bar0 + WRITE_HOST_DA_L_OFFSET));

w_size = ioread32((adapter -> pci_bar0 + WRITE_SIZE_OFFSET));

cst_32 = ioread32((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"w_ddr2: %x; w_addr: %x; w_size: %u; w_cst:%x \\n\

//END, w_counter

w_counter = ioread32((adapter ->

pci_bar0+WRITE_DMA_COUNTER_OFFSET));

printk(KERN_DEBUG \"END, w_counter: %x\\n\

/*************************BAR0 kong jian **************************/

/*************************copy_to_user****************************/

if(unlikely(copy_to_user(buf,virt_addr,count)))

goto err_pci_map;

/*************************copy_to_user****************************/

pci_unmap_single(adapter->pci_dev, dma_write_addr, count,

PCI_DMA_FROMDEVICE);

kfree(virt_addr);

printk(KERN_DEBUG \"Here I am: %s:%i\\n\

return count;

err_pci_map:

//wake_up_interruptible(&adapter->dma_write_wait);

return -1;

err_kmalloc:

return -1;

}

/*file_operation中的write函数。*/

static ssize_t card_write(struct file *file, const char __user *buf, count, loff_t *f_pos)

size_t

{

//unsigned int dma_cst = 0;

int err = -EINVAL;

void * virt_addr = NULL;

dma_addr_t dma_read_addr;

u32 base, r_addr, r_ddr2, r_size, cst_32, r_counter,dma_cst;

struct timeval tv1;

struct timeval tv2;

spinlock_t write_lock;

int i;

i = 0;

/************************BAR0****************************/

pci_read_config_dword(adapter->pci_dev, 0x10, &base);

printk(KERN_DEBUG \" pci_read_config_dword, base:%x\\n\

printk(KERN_DEBUG \"adater -> pci_bar0 %lx\\n\-> pci_bar0));

/************************BAR0****************************/

/***********Request virt_addr(kernel) for Write(DMA Read)*********/

virt_addr = kmalloc(count, GFP_KERNEL|__GFP_DMA);

if(unlikely(copy_from_user(virt_addr, buf, count)))

return err;

printk(KERN_DEBUG \" virt_addr(kernel):%x\\n\

/***********Request virt_addr(kernel) for Write(DMA Read)*********/

/************************dma_read_addr****************************/

dma_read_addr = pci_map_single(adapter->pci_dev, virt_addr, count, PCI_DMA_TODEVICE);

if(unlikely(pci_dma_mapping_error(adapter->pci_dev, dma_read_addr)))

{

//PDEBUG(\"RX DMA MAPPING FAIL...\\n\");

goto err_kmalloc;

}

printk(KERN_DEBUG \" dma_read_addr:%x\\n\

/************************dma_read_addr****************************/

spin_lock_init(&write_lock);

/*************************BAR0 kong jian **************************/

spin_lock(&write_lock);

// START, r_counter

r_counter = ioread32((adapter -> pci_bar0 +

READ_DMA_COUNTER_OFFSET));

printk(KERN_DEBUG \" START, r_counter: %x\

// r_addr

// Lower 32-bit address of system memory buffer for DMA READ operation.

iowrite32(dma_read_addr, READ_HOST_SA_L_OFFSET));

(adapter -> pci_bar0 +

// r_ddr2

iowrite32(*f_pos,(adapter -> pci_bar0 + READ_DDR2_DA_OFFSET));

// r_size

// Read DMA TLP Size Register(from mm to DDR2).

iowrite32(count, (adapter -> pci_bar0 + READ_SIZE_OFFSET));

// dma_cst

// Read DMA Control and Status Register

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst1: %x\

dma_cst = dma_cst | 0x4;

iowrite8(dma_cst,(adapter -> pci_bar0 + DMA_CST_OFFSET));

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst2: %x\

do_gettimeofday(&tv1);

while( (!(dma_cst & 0x8)) && (i < 1000) )

{

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

i++;

//printk(KERN_DEBUG \"******************%d \\n\

}

do_gettimeofday(&tv2);

printk(KERN_DEBUG \"time_time: %ld\

i = 0;

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst3: %x\

dma_cst = dma_cst | 0x8;

iowrite8(dma_cst,(adapter -> pci_bar0 + DMA_CST_OFFSET));

dma_cst = ioread8((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"dma_cst4: %x\

//adapter->dma_read_done = 0;

//if(unlikely(interruptible_sleep_on(adapter->dma_read_wait)))

// goto err_pci_map;

//interruptible_sleep_on(adapter->dma_read_wait);

//test (r_addr,r_ddr2,r_size,cst_32, r_counter)

r_ddr2 = ioread32((adapter -> pci_bar0 + READ_DDR2_DA_OFFSET));

r_addr = ioread32((adapter -> pci_bar0 + READ_HOST_SA_L_OFFSET));

r_size = ioread32((adapter -> pci_bar0 + READ_SIZE_OFFSET));

cst_32 = ioread32((adapter -> pci_bar0 + DMA_CST_OFFSET));

printk(KERN_DEBUG \"r_ddr2: %x; r_addr: %x; r_size: %u; r_cst:%x \\n\

//END, w_counter

r_counter = ioread32((adapter -> pci_bar0 +

READ_DMA_COUNTER_OFFSET));

printk(KERN_DEBUG \"END, r_counter: %x\\n\

spin_unlock(&write_lock);

/*************************BAR0 kong jian **************************/

pci_unmap_single(adapter->pci_dev, PCI_DMA_TODEVICE);

dma_read_addr, count,

kfree(virt_addr);

printk(KERN_DEBUG \"Here I am: %s:%i\\n\

return count;

err_kmalloc:

return -1;

}

static irqreturn_t card_interrupt(int irq, void * dev)

{

u8 status,abc;

status = 0;

printk(KERN_DEBUG \"jinru zhonduan Here am: %s:%i\\n\

//status = card_r32(INT_REG);

//pci_read_config_dword(adapter->pci_dev, DCSR_OFFSET, &status);

status = ioread8((adapter -> pci_bar0+DMA_CST_OFFSET));

//if(!(status & INT_ASSERT))

//{

I

// PDEBUG(\"irq_none ... \\n\");

// return IRQ_NONE;

//}

if(status & INT_ASSERT_W) //DMA Write Complete

{

//card_w32(status, INT_REG);

//clear ISR

//pci_read_config_dword(adapter->pci_dev, DCSR_OFFSET, status);

abc = status | INT_ASSERT_W;

iowrite8(abc,(adapter -> pci_bar0+DMA_CST_OFFSET));

//writel(status,(void __iomem *) (unsigned long) (adapter -> pci_bar0 + DMA_CST_OFFSET));

//i = 1;

//wake_up_interruptible(adapter->dma_write_wait);

flag = 1;

wake_up_interruptible(&dma_write_wait);

return IRQ_HANDLED;

}

if(status & INT_ASSERT_R) //DMA Read Complete

{

//card_w32(status, INT_REG);

//clear ISR

//pci_read_config_dword(adapter->pci_dev, DCSR_OFFSET, status);

status = status & ~INT_ASSERT_R;

writel(status,(void __iomem *) (unsigned long) (adapter -> pci_bar0 + DMA_CST_OFFSET));

//wake_up_interruptible(adapter->dma_read_wait);

return IRQ_HANDLED;

}

//PDEBUG(\"irq handled ... \\n\");

return IRQ_HANDLED;

}

第四节 总结

对于整体构成来说，目前外协实现的功能只包括1.1和1.2两个部分，没有实现1.3部分，也就是操作PCIE设备的内核接口file_operations函数。