1 // Low level AHCI disk access
3 // Copyright (C) 2010 Gerd Hoffmann <kraxel@redhat.com>
5 // This file may be distributed under the terms of the GNU LGPLv3 license.
7 #include "types.h" // u8
8 #include "ioport.h" // inb
9 #include "util.h" // dprintf
10 #include "biosvar.h" // GET_EBDA
11 #include "pci.h" // foreachpci
12 #include "pci_ids.h" // PCI_CLASS_STORAGE_OTHER
13 #include "pci_regs.h" // PCI_INTERRUPT_LINE
14 #include "boot.h" // add_bcv_hd
15 #include "disk.h" // struct ata_s
16 #include "ata.h" // ATA_CB_STAT
17 #include "ahci.h" // CDB_CMD_READ_10
18 #include "blockcmd.h" // CDB_CMD_READ_10
20 #define AHCI_REQUEST_TIMEOUT 32000 // 32 seconds max for IDE ops
21 #define AHCI_RESET_TIMEOUT 500 // 500 miliseconds
22 #define AHCI_LINK_TIMEOUT 10 // 10 miliseconds
24 /****************************************************************
25 * these bits must run in both 16bit and 32bit modes
26 ****************************************************************/
28 // prepare sata command fis
29 static void sata_prep_simple(struct sata_cmd_fis *fis, u8 command)
31 memset_fl(fis, 0, sizeof(*fis));
32 SET_FLATPTR(fis->command, command);
35 static void sata_prep_readwrite(struct sata_cmd_fis *fis,
36 struct disk_op_s *op, int iswrite)
41 memset_fl(fis, 0, sizeof(*fis));
43 if (op->count >= (1<<8) || lba + op->count >= (1<<28)) {
44 SET_FLATPTR(fis->sector_count2, op->count >> 8);
45 SET_FLATPTR(fis->lba_low2, lba >> 24);
46 SET_FLATPTR(fis->lba_mid2, lba >> 32);
47 SET_FLATPTR(fis->lba_high2, lba >> 40);
49 command = (iswrite ? ATA_CMD_WRITE_DMA_EXT
50 : ATA_CMD_READ_DMA_EXT);
52 command = (iswrite ? ATA_CMD_WRITE_DMA
55 SET_FLATPTR(fis->feature, 1); /* dma */
56 SET_FLATPTR(fis->command, command);
57 SET_FLATPTR(fis->sector_count, op->count);
58 SET_FLATPTR(fis->lba_low, lba);
59 SET_FLATPTR(fis->lba_mid, lba >> 8);
60 SET_FLATPTR(fis->lba_high, lba >> 16);
61 SET_FLATPTR(fis->device, ((lba >> 24) & 0xf) | ATA_CB_DH_LBA);
64 static void sata_prep_atapi(struct sata_cmd_fis *fis, u16 blocksize)
66 memset_fl(fis, 0, sizeof(*fis));
67 SET_FLATPTR(fis->command, ATA_CMD_PACKET);
68 SET_FLATPTR(fis->feature, 1); /* dma */
69 SET_FLATPTR(fis->lba_mid, blocksize);
70 SET_FLATPTR(fis->lba_high, blocksize >> 8);
73 // ahci register access helpers
74 static u32 ahci_ctrl_readl(struct ahci_ctrl_s *ctrl, u32 reg)
76 u32 addr = GET_GLOBALFLAT(ctrl->iobase) + reg;
77 return pci_readl(addr);
80 static void ahci_ctrl_writel(struct ahci_ctrl_s *ctrl, u32 reg, u32 val)
82 u32 addr = GET_GLOBALFLAT(ctrl->iobase) + reg;
83 pci_writel(addr, val);
86 static u32 ahci_port_to_ctrl(u32 pnr, u32 port_reg)
89 ctrl_reg += pnr * 0x80;
94 static u32 ahci_port_readl(struct ahci_ctrl_s *ctrl, u32 pnr, u32 reg)
96 u32 ctrl_reg = ahci_port_to_ctrl(pnr, reg);
97 return ahci_ctrl_readl(ctrl, ctrl_reg);
100 static void ahci_port_writel(struct ahci_ctrl_s *ctrl, u32 pnr, u32 reg, u32 val)
102 u32 ctrl_reg = ahci_port_to_ctrl(pnr, reg);
103 ahci_ctrl_writel(ctrl, ctrl_reg, val);
106 // submit ahci command + wait for result
107 static int ahci_command(struct ahci_port_s *port, int iswrite, int isatapi,
108 void *buffer, u32 bsize)
110 u32 val, status, success, flags, intbits, error;
111 struct ahci_ctrl_s *ctrl = GET_GLOBAL(port->ctrl);
112 struct ahci_cmd_s *cmd = GET_GLOBAL(port->cmd);
113 struct ahci_fis_s *fis = GET_GLOBAL(port->fis);
114 struct ahci_list_s *list = GET_GLOBAL(port->list);
115 u32 pnr = GET_GLOBAL(port->pnr);
118 SET_FLATPTR(cmd->fis.reg, 0x27);
119 SET_FLATPTR(cmd->fis.pmp_type, (1 << 7)); /* cmd fis */
120 SET_FLATPTR(cmd->prdt[0].base, ((u32)buffer));
121 SET_FLATPTR(cmd->prdt[0].baseu, 0);
122 SET_FLATPTR(cmd->prdt[0].flags, bsize-1);
124 flags = ((1 << 16) | /* one prd entry */
125 (iswrite ? (1 << 6) : 0) |
126 (isatapi ? (1 << 5) : 0) |
127 (5 << 0)); /* fis length (dwords) */
128 SET_FLATPTR(list[0].flags, flags);
129 SET_FLATPTR(list[0].bytes, 0);
130 SET_FLATPTR(list[0].base, ((u32)(cmd)));
131 SET_FLATPTR(list[0].baseu, 0);
133 dprintf(2, "AHCI/%d: send cmd ...\n", pnr);
134 intbits = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
136 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, intbits);
137 ahci_port_writel(ctrl, pnr, PORT_SCR_ACT, 1);
138 ahci_port_writel(ctrl, pnr, PORT_CMD_ISSUE, 1);
140 end = calc_future_tsc(AHCI_REQUEST_TIMEOUT);
143 intbits = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
145 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, intbits);
146 if (intbits & 0x02) {
147 status = GET_FLATPTR(fis->psfis[2]);
148 error = GET_FLATPTR(fis->psfis[3]);
151 if (intbits & 0x01) {
152 status = GET_FLATPTR(fis->rfis[2]);
153 error = GET_FLATPTR(fis->rfis[3]);
157 if (check_tsc(end)) {
163 dprintf(2, "AHCI/%d: ... intbits 0x%x, status 0x%x ...\n",
164 pnr, intbits, status);
165 } while (status & ATA_CB_STAT_BSY);
167 success = (0x00 == (status & (ATA_CB_STAT_BSY | ATA_CB_STAT_DF |
169 ATA_CB_STAT_RDY == (status & (ATA_CB_STAT_RDY)));
171 dprintf(2, "AHCI/%d: ... finished, status 0x%x, OK\n", pnr,
174 dprintf(2, "AHCI/%d: ... finished, status 0x%x, ERROR 0x%x\n", pnr,
177 // non-queued error recovery (AHCI 1.3 section 6.2.2.1)
178 // Clears PxCMD.ST to 0 to reset the PxCI register
179 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
180 ahci_port_writel(ctrl, pnr, PORT_CMD, val & ~PORT_CMD_START);
182 // waits for PxCMD.CR to clear to 0
184 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
185 if ((val & PORT_CMD_LIST_ON) == 0)
190 // Clears any error bits in PxSERR to enable capturing new errors
191 val = ahci_port_readl(ctrl, pnr, PORT_SCR_ERR);
192 ahci_port_writel(ctrl, pnr, PORT_SCR_ERR, val);
194 // Clears status bits in PxIS as appropriate
195 val = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
196 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, val);
198 // If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to 1, issue
199 // a COMRESET to the device to put it in an idle state
200 val = ahci_port_readl(ctrl, pnr, PORT_TFDATA);
201 if (val & (ATA_CB_STAT_BSY | ATA_CB_STAT_DRQ)) {
202 dprintf(2, "AHCI/%d: issue comreset\n", pnr);
203 val = ahci_port_readl(ctrl, pnr, PORT_SCR_CTL);
204 // set Device Detection Initialization (DET) to 1 for 1 ms for comreset
205 ahci_port_writel(ctrl, pnr, PORT_SCR_CTL, val | 1);
207 ahci_port_writel(ctrl, pnr, PORT_SCR_CTL, val);
210 // Sets PxCMD.ST to 1 to enable issuing new commands
211 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
212 ahci_port_writel(ctrl, pnr, PORT_CMD, val | PORT_CMD_START);
214 return success ? 0 : -1;
217 #define CDROM_CDB_SIZE 12
219 int ahci_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
224 struct ahci_port_s *port = container_of(
225 op->drive_g, struct ahci_port_s, drive);
226 struct ahci_cmd_s *cmd = GET_GLOBAL(port->cmd);
230 sata_prep_atapi(&cmd->fis, blocksize);
231 for (i = 0; i < CDROM_CDB_SIZE; i++) {
232 SET_FLATPTR(cmd->atapi[i], atapi[i]);
234 rc = ahci_command(port, 0, 1, op->buf_fl,
235 op->count * blocksize);
237 return DISK_RET_EBADTRACK;
238 return DISK_RET_SUCCESS;
241 // read/write count blocks from a harddrive, op->buf_fl must be word aligned
243 ahci_disk_readwrite_aligned(struct disk_op_s *op, int iswrite)
245 struct ahci_port_s *port = container_of(
246 op->drive_g, struct ahci_port_s, drive);
247 struct ahci_cmd_s *cmd = GET_GLOBAL(port->cmd);
250 sata_prep_readwrite(&cmd->fis, op, iswrite);
251 rc = ahci_command(port, iswrite, 0, op->buf_fl,
252 op->count * DISK_SECTOR_SIZE);
253 dprintf(2, "ahci disk %s, lba %6x, count %3x, buf %p, rc %d\n",
254 iswrite ? "write" : "read", (u32)op->lba, op->count, op->buf_fl, rc);
256 return DISK_RET_EBADTRACK;
257 return DISK_RET_SUCCESS;
260 // read/write count blocks from a harddrive.
262 ahci_disk_readwrite(struct disk_op_s *op, int iswrite)
264 // if caller's buffer is word aligned, use it directly
265 if (((u32) op->buf_fl & 1) == 0)
266 return ahci_disk_readwrite_aligned(op, iswrite);
268 // Use a word aligned buffer for AHCI I/O
270 struct disk_op_s localop = *op;
271 u8 *alignedbuf_fl = GET_GLOBAL(bounce_buf_fl);
272 u8 *position = op->buf_fl;
274 localop.buf_fl = alignedbuf_fl;
279 for (block = 0; block < op->count; block++) {
280 memcpy_fl (alignedbuf_fl, position, DISK_SECTOR_SIZE);
281 rc = ahci_disk_readwrite_aligned (&localop, 1);
284 position += DISK_SECTOR_SIZE;
289 for (block = 0; block < op->count; block++) {
290 rc = ahci_disk_readwrite_aligned (&localop, 0);
293 memcpy_fl (position, alignedbuf_fl, DISK_SECTOR_SIZE);
294 position += DISK_SECTOR_SIZE;
298 return DISK_RET_SUCCESS;
302 int process_ahci_op(struct disk_op_s *op)
304 struct ahci_port_s *port;
310 port = container_of(op->drive_g, struct ahci_port_s, drive);
311 atapi = GET_GLOBAL(port->atapi);
314 switch (op->command) {
319 return DISK_RET_EWRITEPROTECT;
321 /* FIXME: what should we do here? */
324 return DISK_RET_SUCCESS;
326 dprintf(1, "AHCI: unknown cdrom command %d\n", op->command);
328 return DISK_RET_EPARAM;
331 switch (op->command) {
333 return ahci_disk_readwrite(op, 0);
335 return ahci_disk_readwrite(op, 1);
337 /* FIXME: what should we do here? */
341 return DISK_RET_SUCCESS;
343 dprintf(1, "AHCI: unknown disk command %d\n", op->command);
345 return DISK_RET_EPARAM;
350 /****************************************************************
351 * everything below is pure 32bit code
352 ****************************************************************/
355 ahci_port_reset(struct ahci_ctrl_s *ctrl, u32 pnr)
360 /* disable FIS + CMD */
361 end = calc_future_tsc(AHCI_RESET_TIMEOUT);
363 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
364 if (!(val & (PORT_CMD_FIS_RX | PORT_CMD_START |
365 PORT_CMD_FIS_ON | PORT_CMD_LIST_ON)))
367 val &= ~(PORT_CMD_FIS_RX | PORT_CMD_START);
368 ahci_port_writel(ctrl, pnr, PORT_CMD, val);
369 if (check_tsc(end)) {
376 /* disable + clear IRQs */
377 ahci_port_writel(ctrl, pnr, PORT_IRQ_MASK, 0);
378 val = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
380 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, val);
383 static struct ahci_port_s*
384 ahci_port_alloc(struct ahci_ctrl_s *ctrl, u32 pnr)
386 struct ahci_port_s *port = malloc_tmp(sizeof(*port));
394 port->list = memalign_tmp(1024, 1024);
395 port->fis = memalign_tmp(256, 256);
396 port->cmd = memalign_tmp(256, 256);
397 if (port->list == NULL || port->fis == NULL || port->cmd == NULL) {
401 memset(port->list, 0, 1024);
402 memset(port->fis, 0, 256);
403 memset(port->cmd, 0, 256);
405 ahci_port_writel(ctrl, pnr, PORT_LST_ADDR, (u32)port->list);
406 ahci_port_writel(ctrl, pnr, PORT_FIS_ADDR, (u32)port->fis);
410 static struct ahci_port_s* ahci_port_realloc(struct ahci_port_s *port)
412 struct ahci_port_s *tmp;
415 tmp = malloc_fseg(sizeof(*port));
420 ahci_port_reset(port->ctrl, port->pnr);
425 port->list = memalign_low(1024, 1024);
426 port->fis = memalign_low(256, 256);
427 port->cmd = memalign_low(256, 256);
429 ahci_port_writel(port->ctrl, port->pnr, PORT_LST_ADDR, (u32)port->list);
430 ahci_port_writel(port->ctrl, port->pnr, PORT_FIS_ADDR, (u32)port->fis);
432 cmd = ahci_port_readl(port->ctrl, port->pnr, PORT_CMD);
433 cmd |= (PORT_CMD_FIS_RX|PORT_CMD_START);
434 ahci_port_writel(port->ctrl, port->pnr, PORT_CMD, cmd);
439 static void ahci_port_release(struct ahci_port_s *port)
441 ahci_port_reset(port->ctrl, port->pnr);
450 /* See ahci spec chapter 10.1 "Software Initialization of HBA" */
451 static int ahci_port_init(struct ahci_port_s *port)
453 struct ahci_ctrl_s *ctrl = port->ctrl;
455 char model[MAXMODEL+1];
457 u32 cmd, stat, err, tf;
461 /* enable FIS recv */
462 cmd = ahci_port_readl(ctrl, pnr, PORT_CMD);
463 cmd |= PORT_CMD_FIS_RX;
464 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
467 cmd |= PORT_CMD_SPIN_UP;
468 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
469 end = calc_future_tsc(AHCI_LINK_TIMEOUT);
471 stat = ahci_port_readl(ctrl, pnr, PORT_SCR_STAT);
472 if ((stat & 0x07) == 0x03) {
473 dprintf(1, "AHCI/%d: link up\n", port->pnr);
476 if (check_tsc(end)) {
477 dprintf(1, "AHCI/%d: link down\n", port->pnr);
483 /* clear error status */
484 err = ahci_port_readl(ctrl, pnr, PORT_SCR_ERR);
486 ahci_port_writel(ctrl, pnr, PORT_SCR_ERR, err);
488 /* wait for device becoming ready */
489 end = calc_future_tsc(AHCI_REQUEST_TIMEOUT);
491 tf = ahci_port_readl(ctrl, pnr, PORT_TFDATA);
492 if (!(tf & (ATA_CB_STAT_BSY |
495 if (check_tsc(end)) {
497 dprintf(1, "AHCI/%d: device not ready (tf 0x%x)\n", port->pnr, tf);
504 cmd |= PORT_CMD_START;
505 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
507 sata_prep_simple(&port->cmd->fis, ATA_CMD_IDENTIFY_PACKET_DEVICE);
508 rc = ahci_command(port, 0, 0, buffer, sizeof(buffer));
513 sata_prep_simple(&port->cmd->fis, ATA_CMD_IDENTIFY_DEVICE);
514 rc = ahci_command(port, 0, 0, buffer, sizeof(buffer));
519 port->drive.type = DTYPE_AHCI;
520 port->drive.cntl_id = pnr;
521 port->drive.removable = (buffer[0] & 0x80) ? 1 : 0;
525 port->drive.blksize = DISK_SECTOR_SIZE;
526 port->drive.pchs.cylinders = buffer[1];
527 port->drive.pchs.heads = buffer[3];
528 port->drive.pchs.spt = buffer[6];
531 if (buffer[83] & (1 << 10)) // word 83 - lba48 support
532 sectors = *(u64*)&buffer[100]; // word 100-103
534 sectors = *(u32*)&buffer[60]; // word 60 and word 61
535 port->drive.sectors = sectors;
536 u64 adjsize = sectors >> 11;
537 char adjprefix = 'M';
538 if (adjsize >= (1 << 16)) {
542 port->desc = znprintf(MAXDESCSIZE
543 , "AHCI/%d: %s ATA-%d Hard-Disk (%u %ciBytes)"
545 , ata_extract_model(model, MAXMODEL, buffer)
546 , ata_extract_version(buffer)
547 , (u32)adjsize, adjprefix);
548 port->prio = bootprio_find_ata_device(ctrl->pci_tmp, pnr, 0);
550 // found cdrom (atapi)
551 port->drive.blksize = CDROM_SECTOR_SIZE;
552 port->drive.sectors = (u64)-1;
553 u8 iscd = ((buffer[0] >> 8) & 0x1f) == 0x05;
555 dprintf(1, "AHCI/%d: atapi device is'nt a cdrom\n", port->pnr);
558 port->desc = znprintf(MAXDESCSIZE
559 , "DVD/CD [AHCI/%d: %s ATAPI-%d DVD/CD]"
561 , ata_extract_model(model, MAXMODEL, buffer)
562 , ata_extract_version(buffer));
563 port->prio = bootprio_find_ata_device(ctrl->pci_tmp, pnr, 0);
568 // Detect any drives attached to a given controller.
570 ahci_port_detect(void *data)
572 struct ahci_port_s *port = data;
575 dprintf(2, "AHCI/%d: probing\n", port->pnr);
576 ahci_port_reset(port->ctrl, port->pnr);
577 rc = ahci_port_init(port);
579 ahci_port_release(port);
581 port = ahci_port_realloc(port);
582 dprintf(1, "AHCI/%d: registering: \"%s\"\n", port->pnr, port->desc);
584 // Register with bcv system.
585 boot_add_hd(&port->drive, port->desc, port->prio);
588 boot_add_cd(&port->drive, port->desc, port->prio);
593 // Initialize an ata controller and detect its drives.
595 ahci_init_controller(struct pci_device *pci)
597 struct ahci_ctrl_s *ctrl = malloc_fseg(sizeof(*ctrl));
598 struct ahci_port_s *port;
607 if (bounce_buf_init() < 0) {
615 ctrl->iobase = pci_config_readl(bdf, PCI_BASE_ADDRESS_5);
616 ctrl->irq = pci_config_readb(bdf, PCI_INTERRUPT_LINE);
617 dprintf(1, "AHCI controller at %02x.%x, iobase %x, irq %d\n",
618 bdf >> 3, bdf & 7, ctrl->iobase, ctrl->irq);
620 pci_config_maskw(bdf, PCI_COMMAND, 0,
621 PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
623 val = ahci_ctrl_readl(ctrl, HOST_CTL);
624 ahci_ctrl_writel(ctrl, HOST_CTL, val | HOST_CTL_AHCI_EN);
626 ctrl->caps = ahci_ctrl_readl(ctrl, HOST_CAP);
627 ctrl->ports = ahci_ctrl_readl(ctrl, HOST_PORTS_IMPL);
628 dprintf(2, "AHCI: cap 0x%x, ports_impl 0x%x\n",
629 ctrl->caps, ctrl->ports);
631 max = ctrl->caps & 0x1f;
632 for (pnr = 0; pnr <= max; pnr++) {
633 if (!(ctrl->ports & (1 << pnr)))
635 port = ahci_port_alloc(ctrl, pnr);
638 run_thread(ahci_port_detect, port);
642 // Locate and init ahci controllers.
646 // Scan PCI bus for ATA adapters
647 struct pci_device *pci;
649 if (pci->class != PCI_CLASS_STORAGE_SATA)
651 if (pci->prog_if != 1 /* AHCI rev 1 */)
653 ahci_init_controller(pci);
664 dprintf(3, "init ahci\n");