ia64/linux-2.6.18-xen.hg

view drivers/block/sx8.c @ 897:329ea0ccb344

balloon: try harder to balloon up under memory pressure.

Currently if the balloon driver is unable to increase the guest's
reservation it assumes the failure was due to reaching its full
allocation, gives up on the ballooning operation and records the limit
it reached as the "hard limit". The driver will not try again until
the target is set again (even to the same value).

However it is possible that ballooning has in fact failed due to
memory pressure in the host and therefore it is desirable to keep
attempting to reach the target in case memory becomes available. The
most likely scenario is that some guests are ballooning down while
others are ballooning up and therefore there is temporary memory
pressure while things stabilise. You would not expect a well behaved
toolstack to ask a domain to balloon to more than its allocation nor
would you expect it to deliberately over-commit memory by setting
balloon targets which exceed the total host memory.

This patch drops the concept of a hard limit and causes the balloon
driver to retry increasing the reservation on a timer in the same
manner as when decreasing the reservation.

Also if we partially succeed in increasing the reservation
(i.e. receive less pages than we asked for) then we may as well keep
those pages rather than returning them to Xen.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Jun 05 14:01:20 2009 +0100 (2009-06-05)
parents 831230e53067
children
line source
1 /*
2 * sx8.c: Driver for Promise SATA SX8 looks-like-I2O hardware
3 *
4 * Copyright 2004-2005 Red Hat, Inc.
5 *
6 * Author/maintainer: Jeff Garzik <jgarzik@pobox.com>
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 */
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/pci.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/blkdev.h>
20 #include <linux/sched.h>
21 #include <linux/interrupt.h>
22 #include <linux/compiler.h>
23 #include <linux/workqueue.h>
24 #include <linux/bitops.h>
25 #include <linux/delay.h>
26 #include <linux/time.h>
27 #include <linux/hdreg.h>
28 #include <linux/dma-mapping.h>
29 #include <linux/completion.h>
30 #include <asm/io.h>
31 #include <asm/uaccess.h>
33 #if 0
34 #define CARM_DEBUG
35 #define CARM_VERBOSE_DEBUG
36 #else
37 #undef CARM_DEBUG
38 #undef CARM_VERBOSE_DEBUG
39 #endif
40 #undef CARM_NDEBUG
42 #define DRV_NAME "sx8"
43 #define DRV_VERSION "1.0"
44 #define PFX DRV_NAME ": "
46 MODULE_AUTHOR("Jeff Garzik");
47 MODULE_LICENSE("GPL");
48 MODULE_DESCRIPTION("Promise SATA SX8 block driver");
49 MODULE_VERSION(DRV_VERSION);
51 /*
52 * SX8 hardware has a single message queue for all ATA ports.
53 * When this driver was written, the hardware (firmware?) would
54 * corrupt data eventually, if more than one request was outstanding.
55 * As one can imagine, having 8 ports bottlenecking on a single
56 * command hurts performance.
57 *
58 * Based on user reports, later versions of the hardware (firmware?)
59 * seem to be able to survive with more than one command queued.
60 *
61 * Therefore, we default to the safe option -- 1 command -- but
62 * allow the user to increase this.
63 *
64 * SX8 should be able to support up to ~60 queued commands (CARM_MAX_REQ),
65 * but problems seem to occur when you exceed ~30, even on newer hardware.
66 */
67 static int max_queue = 1;
68 module_param(max_queue, int, 0444);
69 MODULE_PARM_DESC(max_queue, "Maximum number of queued commands. (min==1, max==30, safe==1)");
72 #define NEXT_RESP(idx) ((idx + 1) % RMSG_Q_LEN)
74 /* 0xf is just arbitrary, non-zero noise; this is sorta like poisoning */
75 #define TAG_ENCODE(tag) (((tag) << 16) | 0xf)
76 #define TAG_DECODE(tag) (((tag) >> 16) & 0x1f)
77 #define TAG_VALID(tag) ((((tag) & 0xf) == 0xf) && (TAG_DECODE(tag) < 32))
79 /* note: prints function name for you */
80 #ifdef CARM_DEBUG
81 #define DPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __FUNCTION__, ## args)
82 #ifdef CARM_VERBOSE_DEBUG
83 #define VPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __FUNCTION__, ## args)
84 #else
85 #define VPRINTK(fmt, args...)
86 #endif /* CARM_VERBOSE_DEBUG */
87 #else
88 #define DPRINTK(fmt, args...)
89 #define VPRINTK(fmt, args...)
90 #endif /* CARM_DEBUG */
92 #ifdef CARM_NDEBUG
93 #define assert(expr)
94 #else
95 #define assert(expr) \
96 if(unlikely(!(expr))) { \
97 printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
98 #expr,__FILE__,__FUNCTION__,__LINE__); \
99 }
100 #endif
102 /* defines only for the constants which don't work well as enums */
103 struct carm_host;
105 enum {
106 /* adapter-wide limits */
107 CARM_MAX_PORTS = 8,
108 CARM_SHM_SIZE = (4096 << 7),
109 CARM_MINORS_PER_MAJOR = 256 / CARM_MAX_PORTS,
110 CARM_MAX_WAIT_Q = CARM_MAX_PORTS + 1,
112 /* command message queue limits */
113 CARM_MAX_REQ = 64, /* max command msgs per host */
114 CARM_MSG_LOW_WATER = (CARM_MAX_REQ / 4), /* refill mark */
116 /* S/G limits, host-wide and per-request */
117 CARM_MAX_REQ_SG = 32, /* max s/g entries per request */
118 CARM_MAX_HOST_SG = 600, /* max s/g entries per host */
119 CARM_SG_LOW_WATER = (CARM_MAX_HOST_SG / 4), /* re-fill mark */
121 /* hardware registers */
122 CARM_IHQP = 0x1c,
123 CARM_INT_STAT = 0x10, /* interrupt status */
124 CARM_INT_MASK = 0x14, /* interrupt mask */
125 CARM_HMUC = 0x18, /* host message unit control */
126 RBUF_ADDR_LO = 0x20, /* response msg DMA buf low 32 bits */
127 RBUF_ADDR_HI = 0x24, /* response msg DMA buf high 32 bits */
128 RBUF_BYTE_SZ = 0x28,
129 CARM_RESP_IDX = 0x2c,
130 CARM_CMS0 = 0x30, /* command message size reg 0 */
131 CARM_LMUC = 0x48,
132 CARM_HMPHA = 0x6c,
133 CARM_INITC = 0xb5,
135 /* bits in CARM_INT_{STAT,MASK} */
136 INT_RESERVED = 0xfffffff0,
137 INT_WATCHDOG = (1 << 3), /* watchdog timer */
138 INT_Q_OVERFLOW = (1 << 2), /* cmd msg q overflow */
139 INT_Q_AVAILABLE = (1 << 1), /* cmd msg q has free space */
140 INT_RESPONSE = (1 << 0), /* response msg available */
141 INT_ACK_MASK = INT_WATCHDOG | INT_Q_OVERFLOW,
142 INT_DEF_MASK = INT_RESERVED | INT_Q_OVERFLOW |
143 INT_RESPONSE,
145 /* command messages, and related register bits */
146 CARM_HAVE_RESP = 0x01,
147 CARM_MSG_READ = 1,
148 CARM_MSG_WRITE = 2,
149 CARM_MSG_VERIFY = 3,
150 CARM_MSG_GET_CAPACITY = 4,
151 CARM_MSG_FLUSH = 5,
152 CARM_MSG_IOCTL = 6,
153 CARM_MSG_ARRAY = 8,
154 CARM_MSG_MISC = 9,
155 CARM_CME = (1 << 2),
156 CARM_RME = (1 << 1),
157 CARM_WZBC = (1 << 0),
158 CARM_RMI = (1 << 0),
159 CARM_Q_FULL = (1 << 3),
160 CARM_MSG_SIZE = 288,
161 CARM_Q_LEN = 48,
163 /* CARM_MSG_IOCTL messages */
164 CARM_IOC_SCAN_CHAN = 5, /* scan channels for devices */
165 CARM_IOC_GET_TCQ = 13, /* get tcq/ncq depth */
166 CARM_IOC_SET_TCQ = 14, /* set tcq/ncq depth */
168 IOC_SCAN_CHAN_NODEV = 0x1f,
169 IOC_SCAN_CHAN_OFFSET = 0x40,
171 /* CARM_MSG_ARRAY messages */
172 CARM_ARRAY_INFO = 0,
174 ARRAY_NO_EXIST = (1 << 31),
176 /* response messages */
177 RMSG_SZ = 8, /* sizeof(struct carm_response) */
178 RMSG_Q_LEN = 48, /* resp. msg list length */
179 RMSG_OK = 1, /* bit indicating msg was successful */
180 /* length of entire resp. msg buffer */
181 RBUF_LEN = RMSG_SZ * RMSG_Q_LEN,
183 PDC_SHM_SIZE = (4096 << 7), /* length of entire h/w buffer */
185 /* CARM_MSG_MISC messages */
186 MISC_GET_FW_VER = 2,
187 MISC_ALLOC_MEM = 3,
188 MISC_SET_TIME = 5,
190 /* MISC_GET_FW_VER feature bits */
191 FW_VER_4PORT = (1 << 2), /* 1=4 ports, 0=8 ports */
192 FW_VER_NON_RAID = (1 << 1), /* 1=non-RAID firmware, 0=RAID */
193 FW_VER_ZCR = (1 << 0), /* zero channel RAID (whatever that is) */
195 /* carm_host flags */
196 FL_NON_RAID = FW_VER_NON_RAID,
197 FL_4PORT = FW_VER_4PORT,
198 FL_FW_VER_MASK = (FW_VER_NON_RAID | FW_VER_4PORT),
199 FL_DAC = (1 << 16),
200 FL_DYN_MAJOR = (1 << 17),
201 };
203 enum {
204 CARM_SG_BOUNDARY = 0xffffUL, /* s/g segment boundary */
205 };
207 enum scatter_gather_types {
208 SGT_32BIT = 0,
209 SGT_64BIT = 1,
210 };
212 enum host_states {
213 HST_INVALID, /* invalid state; never used */
214 HST_ALLOC_BUF, /* setting up master SHM area */
215 HST_ERROR, /* we never leave here */
216 HST_PORT_SCAN, /* start dev scan */
217 HST_DEV_SCAN_START, /* start per-device probe */
218 HST_DEV_SCAN, /* continue per-device probe */
219 HST_DEV_ACTIVATE, /* activate devices we found */
220 HST_PROBE_FINISHED, /* probe is complete */
221 HST_PROBE_START, /* initiate probe */
222 HST_SYNC_TIME, /* tell firmware what time it is */
223 HST_GET_FW_VER, /* get firmware version, adapter port cnt */
224 };
226 #ifdef CARM_DEBUG
227 static const char *state_name[] = {
228 "HST_INVALID",
229 "HST_ALLOC_BUF",
230 "HST_ERROR",
231 "HST_PORT_SCAN",
232 "HST_DEV_SCAN_START",
233 "HST_DEV_SCAN",
234 "HST_DEV_ACTIVATE",
235 "HST_PROBE_FINISHED",
236 "HST_PROBE_START",
237 "HST_SYNC_TIME",
238 "HST_GET_FW_VER",
239 };
240 #endif
242 struct carm_port {
243 unsigned int port_no;
244 struct gendisk *disk;
245 struct carm_host *host;
247 /* attached device characteristics */
248 u64 capacity;
249 char name[41];
250 u16 dev_geom_head;
251 u16 dev_geom_sect;
252 u16 dev_geom_cyl;
253 };
255 struct carm_request {
256 unsigned int tag;
257 int n_elem;
258 unsigned int msg_type;
259 unsigned int msg_subtype;
260 unsigned int msg_bucket;
261 struct request *rq;
262 struct carm_port *port;
263 struct scatterlist sg[CARM_MAX_REQ_SG];
264 };
266 struct carm_host {
267 unsigned long flags;
268 void __iomem *mmio;
269 void *shm;
270 dma_addr_t shm_dma;
272 int major;
273 int id;
274 char name[32];
276 spinlock_t lock;
277 struct pci_dev *pdev;
278 unsigned int state;
279 u32 fw_ver;
281 request_queue_t *oob_q;
282 unsigned int n_oob;
284 unsigned int hw_sg_used;
286 unsigned int resp_idx;
288 unsigned int wait_q_prod;
289 unsigned int wait_q_cons;
290 request_queue_t *wait_q[CARM_MAX_WAIT_Q];
292 unsigned int n_msgs;
293 u64 msg_alloc;
294 struct carm_request req[CARM_MAX_REQ];
295 void *msg_base;
296 dma_addr_t msg_dma;
298 int cur_scan_dev;
299 unsigned long dev_active;
300 unsigned long dev_present;
301 struct carm_port port[CARM_MAX_PORTS];
303 struct work_struct fsm_task;
305 struct completion probe_comp;
306 };
308 struct carm_response {
309 __le32 ret_handle;
310 __le32 status;
311 } __attribute__((packed));
313 struct carm_msg_sg {
314 __le32 start;
315 __le32 len;
316 } __attribute__((packed));
318 struct carm_msg_rw {
319 u8 type;
320 u8 id;
321 u8 sg_count;
322 u8 sg_type;
323 __le32 handle;
324 __le32 lba;
325 __le16 lba_count;
326 __le16 lba_high;
327 struct carm_msg_sg sg[32];
328 } __attribute__((packed));
330 struct carm_msg_allocbuf {
331 u8 type;
332 u8 subtype;
333 u8 n_sg;
334 u8 sg_type;
335 __le32 handle;
336 __le32 addr;
337 __le32 len;
338 __le32 evt_pool;
339 __le32 n_evt;
340 __le32 rbuf_pool;
341 __le32 n_rbuf;
342 __le32 msg_pool;
343 __le32 n_msg;
344 struct carm_msg_sg sg[8];
345 } __attribute__((packed));
347 struct carm_msg_ioctl {
348 u8 type;
349 u8 subtype;
350 u8 array_id;
351 u8 reserved1;
352 __le32 handle;
353 __le32 data_addr;
354 u32 reserved2;
355 } __attribute__((packed));
357 struct carm_msg_sync_time {
358 u8 type;
359 u8 subtype;
360 u16 reserved1;
361 __le32 handle;
362 u32 reserved2;
363 __le32 timestamp;
364 } __attribute__((packed));
366 struct carm_msg_get_fw_ver {
367 u8 type;
368 u8 subtype;
369 u16 reserved1;
370 __le32 handle;
371 __le32 data_addr;
372 u32 reserved2;
373 } __attribute__((packed));
375 struct carm_fw_ver {
376 __le32 version;
377 u8 features;
378 u8 reserved1;
379 u16 reserved2;
380 } __attribute__((packed));
382 struct carm_array_info {
383 __le32 size;
385 __le16 size_hi;
386 __le16 stripe_size;
388 __le32 mode;
390 __le16 stripe_blk_sz;
391 __le16 reserved1;
393 __le16 cyl;
394 __le16 head;
396 __le16 sect;
397 u8 array_id;
398 u8 reserved2;
400 char name[40];
402 __le32 array_status;
404 /* device list continues beyond this point? */
405 } __attribute__((packed));
407 static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
408 static void carm_remove_one (struct pci_dev *pdev);
409 static int carm_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo);
411 static struct pci_device_id carm_pci_tbl[] = {
412 { PCI_VENDOR_ID_PROMISE, 0x8000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
413 { PCI_VENDOR_ID_PROMISE, 0x8002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
414 { } /* terminate list */
415 };
416 MODULE_DEVICE_TABLE(pci, carm_pci_tbl);
418 static struct pci_driver carm_driver = {
419 .name = DRV_NAME,
420 .id_table = carm_pci_tbl,
421 .probe = carm_init_one,
422 .remove = carm_remove_one,
423 };
425 static struct block_device_operations carm_bd_ops = {
426 .owner = THIS_MODULE,
427 .getgeo = carm_bdev_getgeo,
428 };
430 static unsigned int carm_host_id;
431 static unsigned long carm_major_alloc;
435 static int carm_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
436 {
437 struct carm_port *port = bdev->bd_disk->private_data;
439 geo->heads = (u8) port->dev_geom_head;
440 geo->sectors = (u8) port->dev_geom_sect;
441 geo->cylinders = port->dev_geom_cyl;
442 return 0;
443 }
445 static const u32 msg_sizes[] = { 32, 64, 128, CARM_MSG_SIZE };
447 static inline int carm_lookup_bucket(u32 msg_size)
448 {
449 int i;
451 for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
452 if (msg_size <= msg_sizes[i])
453 return i;
455 return -ENOENT;
456 }
458 static void carm_init_buckets(void __iomem *mmio)
459 {
460 unsigned int i;
462 for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
463 writel(msg_sizes[i], mmio + CARM_CMS0 + (4 * i));
464 }
466 static inline void *carm_ref_msg(struct carm_host *host,
467 unsigned int msg_idx)
468 {
469 return host->msg_base + (msg_idx * CARM_MSG_SIZE);
470 }
472 static inline dma_addr_t carm_ref_msg_dma(struct carm_host *host,
473 unsigned int msg_idx)
474 {
475 return host->msg_dma + (msg_idx * CARM_MSG_SIZE);
476 }
478 static int carm_send_msg(struct carm_host *host,
479 struct carm_request *crq)
480 {
481 void __iomem *mmio = host->mmio;
482 u32 msg = (u32) carm_ref_msg_dma(host, crq->tag);
483 u32 cm_bucket = crq->msg_bucket;
484 u32 tmp;
485 int rc = 0;
487 VPRINTK("ENTER\n");
489 tmp = readl(mmio + CARM_HMUC);
490 if (tmp & CARM_Q_FULL) {
491 #if 0
492 tmp = readl(mmio + CARM_INT_MASK);
493 tmp |= INT_Q_AVAILABLE;
494 writel(tmp, mmio + CARM_INT_MASK);
495 readl(mmio + CARM_INT_MASK); /* flush */
496 #endif
497 DPRINTK("host msg queue full\n");
498 rc = -EBUSY;
499 } else {
500 writel(msg | (cm_bucket << 1), mmio + CARM_IHQP);
501 readl(mmio + CARM_IHQP); /* flush */
502 }
504 return rc;
505 }
507 static struct carm_request *carm_get_request(struct carm_host *host)
508 {
509 unsigned int i;
511 /* obey global hardware limit on S/G entries */
512 if (host->hw_sg_used >= (CARM_MAX_HOST_SG - CARM_MAX_REQ_SG))
513 return NULL;
515 for (i = 0; i < max_queue; i++)
516 if ((host->msg_alloc & (1ULL << i)) == 0) {
517 struct carm_request *crq = &host->req[i];
518 crq->port = NULL;
519 crq->n_elem = 0;
521 host->msg_alloc |= (1ULL << i);
522 host->n_msgs++;
524 assert(host->n_msgs <= CARM_MAX_REQ);
525 return crq;
526 }
528 DPRINTK("no request available, returning NULL\n");
529 return NULL;
530 }
532 static int carm_put_request(struct carm_host *host, struct carm_request *crq)
533 {
534 assert(crq->tag < max_queue);
536 if (unlikely((host->msg_alloc & (1ULL << crq->tag)) == 0))
537 return -EINVAL; /* tried to clear a tag that was not active */
539 assert(host->hw_sg_used >= crq->n_elem);
541 host->msg_alloc &= ~(1ULL << crq->tag);
542 host->hw_sg_used -= crq->n_elem;
543 host->n_msgs--;
545 return 0;
546 }
548 static struct carm_request *carm_get_special(struct carm_host *host)
549 {
550 unsigned long flags;
551 struct carm_request *crq = NULL;
552 struct request *rq;
553 int tries = 5000;
555 while (tries-- > 0) {
556 spin_lock_irqsave(&host->lock, flags);
557 crq = carm_get_request(host);
558 spin_unlock_irqrestore(&host->lock, flags);
560 if (crq)
561 break;
562 msleep(10);
563 }
565 if (!crq)
566 return NULL;
568 rq = blk_get_request(host->oob_q, WRITE /* bogus */, GFP_KERNEL);
569 if (!rq) {
570 spin_lock_irqsave(&host->lock, flags);
571 carm_put_request(host, crq);
572 spin_unlock_irqrestore(&host->lock, flags);
573 return NULL;
574 }
576 crq->rq = rq;
577 return crq;
578 }
580 static int carm_array_info (struct carm_host *host, unsigned int array_idx)
581 {
582 struct carm_msg_ioctl *ioc;
583 unsigned int idx;
584 u32 msg_data;
585 dma_addr_t msg_dma;
586 struct carm_request *crq;
587 int rc;
589 crq = carm_get_special(host);
590 if (!crq) {
591 rc = -ENOMEM;
592 goto err_out;
593 }
595 idx = crq->tag;
597 ioc = carm_ref_msg(host, idx);
598 msg_dma = carm_ref_msg_dma(host, idx);
599 msg_data = (u32) (msg_dma + sizeof(struct carm_array_info));
601 crq->msg_type = CARM_MSG_ARRAY;
602 crq->msg_subtype = CARM_ARRAY_INFO;
603 rc = carm_lookup_bucket(sizeof(struct carm_msg_ioctl) +
604 sizeof(struct carm_array_info));
605 BUG_ON(rc < 0);
606 crq->msg_bucket = (u32) rc;
608 memset(ioc, 0, sizeof(*ioc));
609 ioc->type = CARM_MSG_ARRAY;
610 ioc->subtype = CARM_ARRAY_INFO;
611 ioc->array_id = (u8) array_idx;
612 ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
613 ioc->data_addr = cpu_to_le32(msg_data);
615 spin_lock_irq(&host->lock);
616 assert(host->state == HST_DEV_SCAN_START ||
617 host->state == HST_DEV_SCAN);
618 spin_unlock_irq(&host->lock);
620 DPRINTK("blk_insert_request, tag == %u\n", idx);
621 blk_insert_request(host->oob_q, crq->rq, 1, crq);
623 return 0;
625 err_out:
626 spin_lock_irq(&host->lock);
627 host->state = HST_ERROR;
628 spin_unlock_irq(&host->lock);
629 return rc;
630 }
632 typedef unsigned int (*carm_sspc_t)(struct carm_host *, unsigned int, void *);
634 static int carm_send_special (struct carm_host *host, carm_sspc_t func)
635 {
636 struct carm_request *crq;
637 struct carm_msg_ioctl *ioc;
638 void *mem;
639 unsigned int idx, msg_size;
640 int rc;
642 crq = carm_get_special(host);
643 if (!crq)
644 return -ENOMEM;
646 idx = crq->tag;
648 mem = carm_ref_msg(host, idx);
650 msg_size = func(host, idx, mem);
652 ioc = mem;
653 crq->msg_type = ioc->type;
654 crq->msg_subtype = ioc->subtype;
655 rc = carm_lookup_bucket(msg_size);
656 BUG_ON(rc < 0);
657 crq->msg_bucket = (u32) rc;
659 DPRINTK("blk_insert_request, tag == %u\n", idx);
660 blk_insert_request(host->oob_q, crq->rq, 1, crq);
662 return 0;
663 }
665 static unsigned int carm_fill_sync_time(struct carm_host *host,
666 unsigned int idx, void *mem)
667 {
668 struct timeval tv;
669 struct carm_msg_sync_time *st = mem;
671 do_gettimeofday(&tv);
673 memset(st, 0, sizeof(*st));
674 st->type = CARM_MSG_MISC;
675 st->subtype = MISC_SET_TIME;
676 st->handle = cpu_to_le32(TAG_ENCODE(idx));
677 st->timestamp = cpu_to_le32(tv.tv_sec);
679 return sizeof(struct carm_msg_sync_time);
680 }
682 static unsigned int carm_fill_alloc_buf(struct carm_host *host,
683 unsigned int idx, void *mem)
684 {
685 struct carm_msg_allocbuf *ab = mem;
687 memset(ab, 0, sizeof(*ab));
688 ab->type = CARM_MSG_MISC;
689 ab->subtype = MISC_ALLOC_MEM;
690 ab->handle = cpu_to_le32(TAG_ENCODE(idx));
691 ab->n_sg = 1;
692 ab->sg_type = SGT_32BIT;
693 ab->addr = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
694 ab->len = cpu_to_le32(PDC_SHM_SIZE >> 1);
695 ab->evt_pool = cpu_to_le32(host->shm_dma + (16 * 1024));
696 ab->n_evt = cpu_to_le32(1024);
697 ab->rbuf_pool = cpu_to_le32(host->shm_dma);
698 ab->n_rbuf = cpu_to_le32(RMSG_Q_LEN);
699 ab->msg_pool = cpu_to_le32(host->shm_dma + RBUF_LEN);
700 ab->n_msg = cpu_to_le32(CARM_Q_LEN);
701 ab->sg[0].start = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
702 ab->sg[0].len = cpu_to_le32(65536);
704 return sizeof(struct carm_msg_allocbuf);
705 }
707 static unsigned int carm_fill_scan_channels(struct carm_host *host,
708 unsigned int idx, void *mem)
709 {
710 struct carm_msg_ioctl *ioc = mem;
711 u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) +
712 IOC_SCAN_CHAN_OFFSET);
714 memset(ioc, 0, sizeof(*ioc));
715 ioc->type = CARM_MSG_IOCTL;
716 ioc->subtype = CARM_IOC_SCAN_CHAN;
717 ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
718 ioc->data_addr = cpu_to_le32(msg_data);
720 /* fill output data area with "no device" default values */
721 mem += IOC_SCAN_CHAN_OFFSET;
722 memset(mem, IOC_SCAN_CHAN_NODEV, CARM_MAX_PORTS);
724 return IOC_SCAN_CHAN_OFFSET + CARM_MAX_PORTS;
725 }
727 static unsigned int carm_fill_get_fw_ver(struct carm_host *host,
728 unsigned int idx, void *mem)
729 {
730 struct carm_msg_get_fw_ver *ioc = mem;
731 u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) + sizeof(*ioc));
733 memset(ioc, 0, sizeof(*ioc));
734 ioc->type = CARM_MSG_MISC;
735 ioc->subtype = MISC_GET_FW_VER;
736 ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
737 ioc->data_addr = cpu_to_le32(msg_data);
739 return sizeof(struct carm_msg_get_fw_ver) +
740 sizeof(struct carm_fw_ver);
741 }
743 static inline void carm_end_request_queued(struct carm_host *host,
744 struct carm_request *crq,
745 int uptodate)
746 {
747 struct request *req = crq->rq;
748 int rc;
750 rc = end_that_request_first(req, uptodate, req->hard_nr_sectors);
751 assert(rc == 0);
753 end_that_request_last(req, uptodate);
755 rc = carm_put_request(host, crq);
756 assert(rc == 0);
757 }
759 static inline void carm_push_q (struct carm_host *host, request_queue_t *q)
760 {
761 unsigned int idx = host->wait_q_prod % CARM_MAX_WAIT_Q;
763 blk_stop_queue(q);
764 VPRINTK("STOPPED QUEUE %p\n", q);
766 host->wait_q[idx] = q;
767 host->wait_q_prod++;
768 BUG_ON(host->wait_q_prod == host->wait_q_cons); /* overrun */
769 }
771 static inline request_queue_t *carm_pop_q(struct carm_host *host)
772 {
773 unsigned int idx;
775 if (host->wait_q_prod == host->wait_q_cons)
776 return NULL;
778 idx = host->wait_q_cons % CARM_MAX_WAIT_Q;
779 host->wait_q_cons++;
781 return host->wait_q[idx];
782 }
784 static inline void carm_round_robin(struct carm_host *host)
785 {
786 request_queue_t *q = carm_pop_q(host);
787 if (q) {
788 blk_start_queue(q);
789 VPRINTK("STARTED QUEUE %p\n", q);
790 }
791 }
793 static inline void carm_end_rq(struct carm_host *host, struct carm_request *crq,
794 int is_ok)
795 {
796 carm_end_request_queued(host, crq, is_ok);
797 if (max_queue == 1)
798 carm_round_robin(host);
799 else if ((host->n_msgs <= CARM_MSG_LOW_WATER) &&
800 (host->hw_sg_used <= CARM_SG_LOW_WATER)) {
801 carm_round_robin(host);
802 }
803 }
805 static void carm_oob_rq_fn(request_queue_t *q)
806 {
807 struct carm_host *host = q->queuedata;
808 struct carm_request *crq;
809 struct request *rq;
810 int rc;
812 while (1) {
813 DPRINTK("get req\n");
814 rq = elv_next_request(q);
815 if (!rq)
816 break;
818 blkdev_dequeue_request(rq);
820 crq = rq->special;
821 assert(crq != NULL);
822 assert(crq->rq == rq);
824 crq->n_elem = 0;
826 DPRINTK("send req\n");
827 rc = carm_send_msg(host, crq);
828 if (rc) {
829 blk_requeue_request(q, rq);
830 carm_push_q(host, q);
831 return; /* call us again later, eventually */
832 }
833 }
834 }
836 static void carm_rq_fn(request_queue_t *q)
837 {
838 struct carm_port *port = q->queuedata;
839 struct carm_host *host = port->host;
840 struct carm_msg_rw *msg;
841 struct carm_request *crq;
842 struct request *rq;
843 struct scatterlist *sg;
844 int writing = 0, pci_dir, i, n_elem, rc;
845 u32 tmp;
846 unsigned int msg_size;
848 queue_one_request:
849 VPRINTK("get req\n");
850 rq = elv_next_request(q);
851 if (!rq)
852 return;
854 crq = carm_get_request(host);
855 if (!crq) {
856 carm_push_q(host, q);
857 return; /* call us again later, eventually */
858 }
859 crq->rq = rq;
861 blkdev_dequeue_request(rq);
863 if (rq_data_dir(rq) == WRITE) {
864 writing = 1;
865 pci_dir = PCI_DMA_TODEVICE;
866 } else {
867 pci_dir = PCI_DMA_FROMDEVICE;
868 }
870 /* get scatterlist from block layer */
871 sg = &crq->sg[0];
872 n_elem = blk_rq_map_sg(q, rq, sg);
873 if (n_elem <= 0) {
874 carm_end_rq(host, crq, 0);
875 return; /* request with no s/g entries? */
876 }
878 /* map scatterlist to PCI bus addresses */
879 n_elem = pci_map_sg(host->pdev, sg, n_elem, pci_dir);
880 if (n_elem <= 0) {
881 carm_end_rq(host, crq, 0);
882 return; /* request with no s/g entries? */
883 }
884 crq->n_elem = n_elem;
885 crq->port = port;
886 host->hw_sg_used += n_elem;
888 /*
889 * build read/write message
890 */
892 VPRINTK("build msg\n");
893 msg = (struct carm_msg_rw *) carm_ref_msg(host, crq->tag);
895 if (writing) {
896 msg->type = CARM_MSG_WRITE;
897 crq->msg_type = CARM_MSG_WRITE;
898 } else {
899 msg->type = CARM_MSG_READ;
900 crq->msg_type = CARM_MSG_READ;
901 }
903 msg->id = port->port_no;
904 msg->sg_count = n_elem;
905 msg->sg_type = SGT_32BIT;
906 msg->handle = cpu_to_le32(TAG_ENCODE(crq->tag));
907 msg->lba = cpu_to_le32(rq->sector & 0xffffffff);
908 tmp = (rq->sector >> 16) >> 16;
909 msg->lba_high = cpu_to_le16( (u16) tmp );
910 msg->lba_count = cpu_to_le16(rq->nr_sectors);
912 msg_size = sizeof(struct carm_msg_rw) - sizeof(msg->sg);
913 for (i = 0; i < n_elem; i++) {
914 struct carm_msg_sg *carm_sg = &msg->sg[i];
915 carm_sg->start = cpu_to_le32(sg_dma_address(&crq->sg[i]));
916 carm_sg->len = cpu_to_le32(sg_dma_len(&crq->sg[i]));
917 msg_size += sizeof(struct carm_msg_sg);
918 }
920 rc = carm_lookup_bucket(msg_size);
921 BUG_ON(rc < 0);
922 crq->msg_bucket = (u32) rc;
924 /*
925 * queue read/write message to hardware
926 */
928 VPRINTK("send msg, tag == %u\n", crq->tag);
929 rc = carm_send_msg(host, crq);
930 if (rc) {
931 carm_put_request(host, crq);
932 blk_requeue_request(q, rq);
933 carm_push_q(host, q);
934 return; /* call us again later, eventually */
935 }
937 goto queue_one_request;
938 }
940 static void carm_handle_array_info(struct carm_host *host,
941 struct carm_request *crq, u8 *mem,
942 int is_ok)
943 {
944 struct carm_port *port;
945 u8 *msg_data = mem + sizeof(struct carm_array_info);
946 struct carm_array_info *desc = (struct carm_array_info *) msg_data;
947 u64 lo, hi;
948 int cur_port;
949 size_t slen;
951 DPRINTK("ENTER\n");
953 carm_end_rq(host, crq, is_ok);
955 if (!is_ok)
956 goto out;
957 if (le32_to_cpu(desc->array_status) & ARRAY_NO_EXIST)
958 goto out;
960 cur_port = host->cur_scan_dev;
962 /* should never occur */
963 if ((cur_port < 0) || (cur_port >= CARM_MAX_PORTS)) {
964 printk(KERN_ERR PFX "BUG: cur_scan_dev==%d, array_id==%d\n",
965 cur_port, (int) desc->array_id);
966 goto out;
967 }
969 port = &host->port[cur_port];
971 lo = (u64) le32_to_cpu(desc->size);
972 hi = (u64) le16_to_cpu(desc->size_hi);
974 port->capacity = lo | (hi << 32);
975 port->dev_geom_head = le16_to_cpu(desc->head);
976 port->dev_geom_sect = le16_to_cpu(desc->sect);
977 port->dev_geom_cyl = le16_to_cpu(desc->cyl);
979 host->dev_active |= (1 << cur_port);
981 strncpy(port->name, desc->name, sizeof(port->name));
982 port->name[sizeof(port->name) - 1] = 0;
983 slen = strlen(port->name);
984 while (slen && (port->name[slen - 1] == ' ')) {
985 port->name[slen - 1] = 0;
986 slen--;
987 }
989 printk(KERN_INFO DRV_NAME "(%s): port %u device %Lu sectors\n",
990 pci_name(host->pdev), port->port_no,
991 (unsigned long long) port->capacity);
992 printk(KERN_INFO DRV_NAME "(%s): port %u device \"%s\"\n",
993 pci_name(host->pdev), port->port_no, port->name);
995 out:
996 assert(host->state == HST_DEV_SCAN);
997 schedule_work(&host->fsm_task);
998 }
1000 static void carm_handle_scan_chan(struct carm_host *host,
1001 struct carm_request *crq, u8 *mem,
1002 int is_ok)
1004 u8 *msg_data = mem + IOC_SCAN_CHAN_OFFSET;
1005 unsigned int i, dev_count = 0;
1006 int new_state = HST_DEV_SCAN_START;
1008 DPRINTK("ENTER\n");
1010 carm_end_rq(host, crq, is_ok);
1012 if (!is_ok) {
1013 new_state = HST_ERROR;
1014 goto out;
1017 /* TODO: scan and support non-disk devices */
1018 for (i = 0; i < 8; i++)
1019 if (msg_data[i] == 0) { /* direct-access device (disk) */
1020 host->dev_present |= (1 << i);
1021 dev_count++;
1024 printk(KERN_INFO DRV_NAME "(%s): found %u interesting devices\n",
1025 pci_name(host->pdev), dev_count);
1027 out:
1028 assert(host->state == HST_PORT_SCAN);
1029 host->state = new_state;
1030 schedule_work(&host->fsm_task);
1033 static void carm_handle_generic(struct carm_host *host,
1034 struct carm_request *crq, int is_ok,
1035 int cur_state, int next_state)
1037 DPRINTK("ENTER\n");
1039 carm_end_rq(host, crq, is_ok);
1041 assert(host->state == cur_state);
1042 if (is_ok)
1043 host->state = next_state;
1044 else
1045 host->state = HST_ERROR;
1046 schedule_work(&host->fsm_task);
1049 static inline void carm_handle_rw(struct carm_host *host,
1050 struct carm_request *crq, int is_ok)
1052 int pci_dir;
1054 VPRINTK("ENTER\n");
1056 if (rq_data_dir(crq->rq) == WRITE)
1057 pci_dir = PCI_DMA_TODEVICE;
1058 else
1059 pci_dir = PCI_DMA_FROMDEVICE;
1061 pci_unmap_sg(host->pdev, &crq->sg[0], crq->n_elem, pci_dir);
1063 carm_end_rq(host, crq, is_ok);
1066 static inline void carm_handle_resp(struct carm_host *host,
1067 __le32 ret_handle_le, u32 status)
1069 u32 handle = le32_to_cpu(ret_handle_le);
1070 unsigned int msg_idx;
1071 struct carm_request *crq;
1072 int is_ok = (status == RMSG_OK);
1073 u8 *mem;
1075 VPRINTK("ENTER, handle == 0x%x\n", handle);
1077 if (unlikely(!TAG_VALID(handle))) {
1078 printk(KERN_ERR DRV_NAME "(%s): BUG: invalid tag 0x%x\n",
1079 pci_name(host->pdev), handle);
1080 return;
1083 msg_idx = TAG_DECODE(handle);
1084 VPRINTK("tag == %u\n", msg_idx);
1086 crq = &host->req[msg_idx];
1088 /* fast path */
1089 if (likely(crq->msg_type == CARM_MSG_READ ||
1090 crq->msg_type == CARM_MSG_WRITE)) {
1091 carm_handle_rw(host, crq, is_ok);
1092 return;
1095 mem = carm_ref_msg(host, msg_idx);
1097 switch (crq->msg_type) {
1098 case CARM_MSG_IOCTL: {
1099 switch (crq->msg_subtype) {
1100 case CARM_IOC_SCAN_CHAN:
1101 carm_handle_scan_chan(host, crq, mem, is_ok);
1102 break;
1103 default:
1104 /* unknown / invalid response */
1105 goto err_out;
1107 break;
1110 case CARM_MSG_MISC: {
1111 switch (crq->msg_subtype) {
1112 case MISC_ALLOC_MEM:
1113 carm_handle_generic(host, crq, is_ok,
1114 HST_ALLOC_BUF, HST_SYNC_TIME);
1115 break;
1116 case MISC_SET_TIME:
1117 carm_handle_generic(host, crq, is_ok,
1118 HST_SYNC_TIME, HST_GET_FW_VER);
1119 break;
1120 case MISC_GET_FW_VER: {
1121 struct carm_fw_ver *ver = (struct carm_fw_ver *)
1122 mem + sizeof(struct carm_msg_get_fw_ver);
1123 if (is_ok) {
1124 host->fw_ver = le32_to_cpu(ver->version);
1125 host->flags |= (ver->features & FL_FW_VER_MASK);
1127 carm_handle_generic(host, crq, is_ok,
1128 HST_GET_FW_VER, HST_PORT_SCAN);
1129 break;
1131 default:
1132 /* unknown / invalid response */
1133 goto err_out;
1135 break;
1138 case CARM_MSG_ARRAY: {
1139 switch (crq->msg_subtype) {
1140 case CARM_ARRAY_INFO:
1141 carm_handle_array_info(host, crq, mem, is_ok);
1142 break;
1143 default:
1144 /* unknown / invalid response */
1145 goto err_out;
1147 break;
1150 default:
1151 /* unknown / invalid response */
1152 goto err_out;
1155 return;
1157 err_out:
1158 printk(KERN_WARNING DRV_NAME "(%s): BUG: unhandled message type %d/%d\n",
1159 pci_name(host->pdev), crq->msg_type, crq->msg_subtype);
1160 carm_end_rq(host, crq, 0);
1163 static inline void carm_handle_responses(struct carm_host *host)
1165 void __iomem *mmio = host->mmio;
1166 struct carm_response *resp = (struct carm_response *) host->shm;
1167 unsigned int work = 0;
1168 unsigned int idx = host->resp_idx % RMSG_Q_LEN;
1170 while (1) {
1171 u32 status = le32_to_cpu(resp[idx].status);
1173 if (status == 0xffffffff) {
1174 VPRINTK("ending response on index %u\n", idx);
1175 writel(idx << 3, mmio + CARM_RESP_IDX);
1176 break;
1179 /* response to a message we sent */
1180 else if ((status & (1 << 31)) == 0) {
1181 VPRINTK("handling msg response on index %u\n", idx);
1182 carm_handle_resp(host, resp[idx].ret_handle, status);
1183 resp[idx].status = cpu_to_le32(0xffffffff);
1186 /* asynchronous events the hardware throws our way */
1187 else if ((status & 0xff000000) == (1 << 31)) {
1188 u8 *evt_type_ptr = (u8 *) &resp[idx];
1189 u8 evt_type = *evt_type_ptr;
1190 printk(KERN_WARNING DRV_NAME "(%s): unhandled event type %d\n",
1191 pci_name(host->pdev), (int) evt_type);
1192 resp[idx].status = cpu_to_le32(0xffffffff);
1195 idx = NEXT_RESP(idx);
1196 work++;
1199 VPRINTK("EXIT, work==%u\n", work);
1200 host->resp_idx += work;
1203 static irqreturn_t carm_interrupt(int irq, void *__host, struct pt_regs *regs)
1205 struct carm_host *host = __host;
1206 void __iomem *mmio;
1207 u32 mask;
1208 int handled = 0;
1209 unsigned long flags;
1211 if (!host) {
1212 VPRINTK("no host\n");
1213 return IRQ_NONE;
1216 spin_lock_irqsave(&host->lock, flags);
1218 mmio = host->mmio;
1220 /* reading should also clear interrupts */
1221 mask = readl(mmio + CARM_INT_STAT);
1223 if (mask == 0 || mask == 0xffffffff) {
1224 VPRINTK("no work, mask == 0x%x\n", mask);
1225 goto out;
1228 if (mask & INT_ACK_MASK)
1229 writel(mask, mmio + CARM_INT_STAT);
1231 if (unlikely(host->state == HST_INVALID)) {
1232 VPRINTK("not initialized yet, mask = 0x%x\n", mask);
1233 goto out;
1236 if (mask & CARM_HAVE_RESP) {
1237 handled = 1;
1238 carm_handle_responses(host);
1241 out:
1242 spin_unlock_irqrestore(&host->lock, flags);
1243 VPRINTK("EXIT\n");
1244 return IRQ_RETVAL(handled);
1247 static void carm_fsm_task (void *_data)
1249 struct carm_host *host = _data;
1250 unsigned long flags;
1251 unsigned int state;
1252 int rc, i, next_dev;
1253 int reschedule = 0;
1254 int new_state = HST_INVALID;
1256 spin_lock_irqsave(&host->lock, flags);
1257 state = host->state;
1258 spin_unlock_irqrestore(&host->lock, flags);
1260 DPRINTK("ENTER, state == %s\n", state_name[state]);
1262 switch (state) {
1263 case HST_PROBE_START:
1264 new_state = HST_ALLOC_BUF;
1265 reschedule = 1;
1266 break;
1268 case HST_ALLOC_BUF:
1269 rc = carm_send_special(host, carm_fill_alloc_buf);
1270 if (rc) {
1271 new_state = HST_ERROR;
1272 reschedule = 1;
1274 break;
1276 case HST_SYNC_TIME:
1277 rc = carm_send_special(host, carm_fill_sync_time);
1278 if (rc) {
1279 new_state = HST_ERROR;
1280 reschedule = 1;
1282 break;
1284 case HST_GET_FW_VER:
1285 rc = carm_send_special(host, carm_fill_get_fw_ver);
1286 if (rc) {
1287 new_state = HST_ERROR;
1288 reschedule = 1;
1290 break;
1292 case HST_PORT_SCAN:
1293 rc = carm_send_special(host, carm_fill_scan_channels);
1294 if (rc) {
1295 new_state = HST_ERROR;
1296 reschedule = 1;
1298 break;
1300 case HST_DEV_SCAN_START:
1301 host->cur_scan_dev = -1;
1302 new_state = HST_DEV_SCAN;
1303 reschedule = 1;
1304 break;
1306 case HST_DEV_SCAN:
1307 next_dev = -1;
1308 for (i = host->cur_scan_dev + 1; i < CARM_MAX_PORTS; i++)
1309 if (host->dev_present & (1 << i)) {
1310 next_dev = i;
1311 break;
1314 if (next_dev >= 0) {
1315 host->cur_scan_dev = next_dev;
1316 rc = carm_array_info(host, next_dev);
1317 if (rc) {
1318 new_state = HST_ERROR;
1319 reschedule = 1;
1321 } else {
1322 new_state = HST_DEV_ACTIVATE;
1323 reschedule = 1;
1325 break;
1327 case HST_DEV_ACTIVATE: {
1328 int activated = 0;
1329 for (i = 0; i < CARM_MAX_PORTS; i++)
1330 if (host->dev_active & (1 << i)) {
1331 struct carm_port *port = &host->port[i];
1332 struct gendisk *disk = port->disk;
1334 set_capacity(disk, port->capacity);
1335 add_disk(disk);
1336 activated++;
1339 printk(KERN_INFO DRV_NAME "(%s): %d ports activated\n",
1340 pci_name(host->pdev), activated);
1342 new_state = HST_PROBE_FINISHED;
1343 reschedule = 1;
1344 break;
1347 case HST_PROBE_FINISHED:
1348 complete(&host->probe_comp);
1349 break;
1351 case HST_ERROR:
1352 /* FIXME: TODO */
1353 break;
1355 default:
1356 /* should never occur */
1357 printk(KERN_ERR PFX "BUG: unknown state %d\n", state);
1358 assert(0);
1359 break;
1362 if (new_state != HST_INVALID) {
1363 spin_lock_irqsave(&host->lock, flags);
1364 host->state = new_state;
1365 spin_unlock_irqrestore(&host->lock, flags);
1367 if (reschedule)
1368 schedule_work(&host->fsm_task);
1371 static int carm_init_wait(void __iomem *mmio, u32 bits, unsigned int test_bit)
1373 unsigned int i;
1375 for (i = 0; i < 50000; i++) {
1376 u32 tmp = readl(mmio + CARM_LMUC);
1377 udelay(100);
1379 if (test_bit) {
1380 if ((tmp & bits) == bits)
1381 return 0;
1382 } else {
1383 if ((tmp & bits) == 0)
1384 return 0;
1387 cond_resched();
1390 printk(KERN_ERR PFX "carm_init_wait timeout, bits == 0x%x, test_bit == %s\n",
1391 bits, test_bit ? "yes" : "no");
1392 return -EBUSY;
1395 static void carm_init_responses(struct carm_host *host)
1397 void __iomem *mmio = host->mmio;
1398 unsigned int i;
1399 struct carm_response *resp = (struct carm_response *) host->shm;
1401 for (i = 0; i < RMSG_Q_LEN; i++)
1402 resp[i].status = cpu_to_le32(0xffffffff);
1404 writel(0, mmio + CARM_RESP_IDX);
1407 static int carm_init_host(struct carm_host *host)
1409 void __iomem *mmio = host->mmio;
1410 u32 tmp;
1411 u8 tmp8;
1412 int rc;
1414 DPRINTK("ENTER\n");
1416 writel(0, mmio + CARM_INT_MASK);
1418 tmp8 = readb(mmio + CARM_INITC);
1419 if (tmp8 & 0x01) {
1420 tmp8 &= ~0x01;
1421 writeb(tmp8, mmio + CARM_INITC);
1422 readb(mmio + CARM_INITC); /* flush */
1424 DPRINTK("snooze...\n");
1425 msleep(5000);
1428 tmp = readl(mmio + CARM_HMUC);
1429 if (tmp & CARM_CME) {
1430 DPRINTK("CME bit present, waiting\n");
1431 rc = carm_init_wait(mmio, CARM_CME, 1);
1432 if (rc) {
1433 DPRINTK("EXIT, carm_init_wait 1 failed\n");
1434 return rc;
1437 if (tmp & CARM_RME) {
1438 DPRINTK("RME bit present, waiting\n");
1439 rc = carm_init_wait(mmio, CARM_RME, 1);
1440 if (rc) {
1441 DPRINTK("EXIT, carm_init_wait 2 failed\n");
1442 return rc;
1446 tmp &= ~(CARM_RME | CARM_CME);
1447 writel(tmp, mmio + CARM_HMUC);
1448 readl(mmio + CARM_HMUC); /* flush */
1450 rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 0);
1451 if (rc) {
1452 DPRINTK("EXIT, carm_init_wait 3 failed\n");
1453 return rc;
1456 carm_init_buckets(mmio);
1458 writel(host->shm_dma & 0xffffffff, mmio + RBUF_ADDR_LO);
1459 writel((host->shm_dma >> 16) >> 16, mmio + RBUF_ADDR_HI);
1460 writel(RBUF_LEN, mmio + RBUF_BYTE_SZ);
1462 tmp = readl(mmio + CARM_HMUC);
1463 tmp |= (CARM_RME | CARM_CME | CARM_WZBC);
1464 writel(tmp, mmio + CARM_HMUC);
1465 readl(mmio + CARM_HMUC); /* flush */
1467 rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 1);
1468 if (rc) {
1469 DPRINTK("EXIT, carm_init_wait 4 failed\n");
1470 return rc;
1473 writel(0, mmio + CARM_HMPHA);
1474 writel(INT_DEF_MASK, mmio + CARM_INT_MASK);
1476 carm_init_responses(host);
1478 /* start initialization, probing state machine */
1479 spin_lock_irq(&host->lock);
1480 assert(host->state == HST_INVALID);
1481 host->state = HST_PROBE_START;
1482 spin_unlock_irq(&host->lock);
1483 schedule_work(&host->fsm_task);
1485 DPRINTK("EXIT\n");
1486 return 0;
1489 static int carm_init_disks(struct carm_host *host)
1491 unsigned int i;
1492 int rc = 0;
1494 for (i = 0; i < CARM_MAX_PORTS; i++) {
1495 struct gendisk *disk;
1496 request_queue_t *q;
1497 struct carm_port *port;
1499 port = &host->port[i];
1500 port->host = host;
1501 port->port_no = i;
1503 disk = alloc_disk(CARM_MINORS_PER_MAJOR);
1504 if (!disk) {
1505 rc = -ENOMEM;
1506 break;
1509 port->disk = disk;
1510 sprintf(disk->disk_name, DRV_NAME "/%u",
1511 (unsigned int) (host->id * CARM_MAX_PORTS) + i);
1512 disk->major = host->major;
1513 disk->first_minor = i * CARM_MINORS_PER_MAJOR;
1514 disk->fops = &carm_bd_ops;
1515 disk->private_data = port;
1517 q = blk_init_queue(carm_rq_fn, &host->lock);
1518 if (!q) {
1519 rc = -ENOMEM;
1520 break;
1522 disk->queue = q;
1523 blk_queue_max_hw_segments(q, CARM_MAX_REQ_SG);
1524 blk_queue_max_phys_segments(q, CARM_MAX_REQ_SG);
1525 blk_queue_segment_boundary(q, CARM_SG_BOUNDARY);
1527 q->queuedata = port;
1530 return rc;
1533 static void carm_free_disks(struct carm_host *host)
1535 unsigned int i;
1537 for (i = 0; i < CARM_MAX_PORTS; i++) {
1538 struct gendisk *disk = host->port[i].disk;
1539 if (disk) {
1540 request_queue_t *q = disk->queue;
1542 if (disk->flags & GENHD_FL_UP)
1543 del_gendisk(disk);
1544 if (q)
1545 blk_cleanup_queue(q);
1546 put_disk(disk);
1551 static int carm_init_shm(struct carm_host *host)
1553 host->shm = pci_alloc_consistent(host->pdev, CARM_SHM_SIZE,
1554 &host->shm_dma);
1555 if (!host->shm)
1556 return -ENOMEM;
1558 host->msg_base = host->shm + RBUF_LEN;
1559 host->msg_dma = host->shm_dma + RBUF_LEN;
1561 memset(host->shm, 0xff, RBUF_LEN);
1562 memset(host->msg_base, 0, PDC_SHM_SIZE - RBUF_LEN);
1564 return 0;
1567 static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1569 static unsigned int printed_version;
1570 struct carm_host *host;
1571 unsigned int pci_dac;
1572 int rc;
1573 request_queue_t *q;
1574 unsigned int i;
1576 if (!printed_version++)
1577 printk(KERN_DEBUG DRV_NAME " version " DRV_VERSION "\n");
1579 rc = pci_enable_device(pdev);
1580 if (rc)
1581 return rc;
1583 rc = pci_request_regions(pdev, DRV_NAME);
1584 if (rc)
1585 goto err_out;
1587 #ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */
1588 rc = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
1589 if (!rc) {
1590 rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1591 if (rc) {
1592 printk(KERN_ERR DRV_NAME "(%s): consistent DMA mask failure\n",
1593 pci_name(pdev));
1594 goto err_out_regions;
1596 pci_dac = 1;
1597 } else {
1598 #endif
1599 rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1600 if (rc) {
1601 printk(KERN_ERR DRV_NAME "(%s): DMA mask failure\n",
1602 pci_name(pdev));
1603 goto err_out_regions;
1605 pci_dac = 0;
1606 #ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */
1608 #endif
1610 host = kmalloc(sizeof(*host), GFP_KERNEL);
1611 if (!host) {
1612 printk(KERN_ERR DRV_NAME "(%s): memory alloc failure\n",
1613 pci_name(pdev));
1614 rc = -ENOMEM;
1615 goto err_out_regions;
1618 memset(host, 0, sizeof(*host));
1619 host->pdev = pdev;
1620 host->flags = pci_dac ? FL_DAC : 0;
1621 spin_lock_init(&host->lock);
1622 INIT_WORK(&host->fsm_task, carm_fsm_task, host);
1623 init_completion(&host->probe_comp);
1625 for (i = 0; i < ARRAY_SIZE(host->req); i++)
1626 host->req[i].tag = i;
1628 host->mmio = ioremap(pci_resource_start(pdev, 0),
1629 pci_resource_len(pdev, 0));
1630 if (!host->mmio) {
1631 printk(KERN_ERR DRV_NAME "(%s): MMIO alloc failure\n",
1632 pci_name(pdev));
1633 rc = -ENOMEM;
1634 goto err_out_kfree;
1637 rc = carm_init_shm(host);
1638 if (rc) {
1639 printk(KERN_ERR DRV_NAME "(%s): DMA SHM alloc failure\n",
1640 pci_name(pdev));
1641 goto err_out_iounmap;
1644 q = blk_init_queue(carm_oob_rq_fn, &host->lock);
1645 if (!q) {
1646 printk(KERN_ERR DRV_NAME "(%s): OOB queue alloc failure\n",
1647 pci_name(pdev));
1648 rc = -ENOMEM;
1649 goto err_out_pci_free;
1651 host->oob_q = q;
1652 q->queuedata = host;
1654 /*
1655 * Figure out which major to use: 160, 161, or dynamic
1656 */
1657 if (!test_and_set_bit(0, &carm_major_alloc))
1658 host->major = 160;
1659 else if (!test_and_set_bit(1, &carm_major_alloc))
1660 host->major = 161;
1661 else
1662 host->flags |= FL_DYN_MAJOR;
1664 host->id = carm_host_id;
1665 sprintf(host->name, DRV_NAME "%d", carm_host_id);
1667 rc = register_blkdev(host->major, host->name);
1668 if (rc < 0)
1669 goto err_out_free_majors;
1670 if (host->flags & FL_DYN_MAJOR)
1671 host->major = rc;
1673 rc = carm_init_disks(host);
1674 if (rc)
1675 goto err_out_blkdev_disks;
1677 pci_set_master(pdev);
1679 rc = request_irq(pdev->irq, carm_interrupt, IRQF_SHARED, DRV_NAME, host);
1680 if (rc) {
1681 printk(KERN_ERR DRV_NAME "(%s): irq alloc failure\n",
1682 pci_name(pdev));
1683 goto err_out_blkdev_disks;
1686 rc = carm_init_host(host);
1687 if (rc)
1688 goto err_out_free_irq;
1690 DPRINTK("waiting for probe_comp\n");
1691 wait_for_completion(&host->probe_comp);
1693 printk(KERN_INFO "%s: pci %s, ports %d, io %llx, irq %u, major %d\n",
1694 host->name, pci_name(pdev), (int) CARM_MAX_PORTS,
1695 (unsigned long long)pci_resource_start(pdev, 0),
1696 pdev->irq, host->major);
1698 carm_host_id++;
1699 pci_set_drvdata(pdev, host);
1700 return 0;
1702 err_out_free_irq:
1703 free_irq(pdev->irq, host);
1704 err_out_blkdev_disks:
1705 carm_free_disks(host);
1706 unregister_blkdev(host->major, host->name);
1707 err_out_free_majors:
1708 if (host->major == 160)
1709 clear_bit(0, &carm_major_alloc);
1710 else if (host->major == 161)
1711 clear_bit(1, &carm_major_alloc);
1712 blk_cleanup_queue(host->oob_q);
1713 err_out_pci_free:
1714 pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma);
1715 err_out_iounmap:
1716 iounmap(host->mmio);
1717 err_out_kfree:
1718 kfree(host);
1719 err_out_regions:
1720 pci_release_regions(pdev);
1721 err_out:
1722 pci_disable_device(pdev);
1723 return rc;
1726 static void carm_remove_one (struct pci_dev *pdev)
1728 struct carm_host *host = pci_get_drvdata(pdev);
1730 if (!host) {
1731 printk(KERN_ERR PFX "BUG: no host data for PCI(%s)\n",
1732 pci_name(pdev));
1733 return;
1736 free_irq(pdev->irq, host);
1737 carm_free_disks(host);
1738 unregister_blkdev(host->major, host->name);
1739 if (host->major == 160)
1740 clear_bit(0, &carm_major_alloc);
1741 else if (host->major == 161)
1742 clear_bit(1, &carm_major_alloc);
1743 blk_cleanup_queue(host->oob_q);
1744 pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma);
1745 iounmap(host->mmio);
1746 kfree(host);
1747 pci_release_regions(pdev);
1748 pci_disable_device(pdev);
1749 pci_set_drvdata(pdev, NULL);
1752 static int __init carm_init(void)
1754 return pci_register_driver(&carm_driver);
1757 static void __exit carm_exit(void)
1759 pci_unregister_driver(&carm_driver);
1762 module_init(carm_init);
1763 module_exit(carm_exit);