// See LICENSE for license details.
#include "mmu.h"
-#include "sim.h"
+#include "simif.h"
#include "processor.h"
-#define LEVELS(xlen) ((xlen) == 32 ? 2 : 3)
-#define PPN_SHIFT(xlen) ((xlen) == 32 ? 10 : 26)
-#define PTIDXBITS(xlen) ((xlen) == 32 ? 10 : 9)
-#define VPN_BITS(xlen) (PTIDXBITS(xlen) * LEVELS(xlen))
-#define VA_BITS(xlen) (VPN_BITS(xlen) + PGSHIFT)
-
-mmu_t::mmu_t(char* _mem, size_t _memsz)
- : mem(_mem), memsz(_memsz), proc(NULL)
+mmu_t::mmu_t(simif_t* sim, processor_t* proc)
+ : sim(sim), proc(proc),
+ check_triggers_fetch(false),
+ check_triggers_load(false),
+ check_triggers_store(false),
+ matched_trigger(NULL)
{
flush_tlb();
}
flush_icache();
}
-void* mmu_t::refill_tlb(reg_t addr, reg_t bytes, bool store, bool fetch)
+reg_t mmu_t::translate(reg_t addr, access_type type)
+{
+ if (!proc)
+ return addr;
+
+ reg_t mode = proc->state.prv;
+ if (type != FETCH) {
+ if (!proc->state.dcsr.cause && get_field(proc->state.mstatus, MSTATUS_MPRV))
+ mode = get_field(proc->state.mstatus, MSTATUS_MPP);
+ }
+
+ return walk(addr, type, mode) | (addr & (PGSIZE-1));
+}
+
+tlb_entry_t mmu_t::fetch_slow_path(reg_t vaddr)
{
- reg_t idx = (addr >> PGSHIFT) % TLB_ENTRIES;
- reg_t expected_tag = addr >> PGSHIFT;
-
- reg_t mstatus = proc ? proc->state.mstatus : 0;
-
- bool vm_disabled = get_field(mstatus, MSTATUS_VM) == VM_MBARE;
- bool mode_m = get_field(mstatus, MSTATUS_PRV) == PRV_M;
- bool mode_s = get_field(mstatus, MSTATUS_PRV) == PRV_S;
- bool mprv_m = get_field(mstatus, MSTATUS_MPRV) == PRV_M;
- bool mprv_s = get_field(mstatus, MSTATUS_MPRV) == PRV_S;
-
- reg_t pgbase;
- if (vm_disabled || (mode_m && (mprv_m || fetch))) {
- pgbase = addr & -PGSIZE;
- // virtual memory is disabled. merely check legality of physical address.
- if (addr >= memsz)
- pgbase = -1;
+ reg_t paddr = translate(vaddr, FETCH);
+
+ if (auto host_addr = sim->addr_to_mem(paddr)) {
+ return refill_tlb(vaddr, paddr, host_addr, FETCH);
} else {
- pgbase = walk(addr, mode_s || (mode_m && mprv_s), store, fetch);
+ if (!sim->mmio_load(paddr, sizeof fetch_temp, (uint8_t*)&fetch_temp))
+ throw trap_instruction_access_fault(vaddr);
+ tlb_entry_t entry = {(char*)&fetch_temp - vaddr, paddr - vaddr};
+ return entry;
}
+}
- reg_t pgoff = addr & (PGSIZE-1);
- reg_t paddr = pgbase + pgoff;
+reg_t reg_from_bytes(size_t len, const uint8_t* bytes)
+{
+ switch (len) {
+ case 1:
+ return bytes[0];
+ case 2:
+ return bytes[0] |
+ (((reg_t) bytes[1]) << 8);
+ case 4:
+ return bytes[0] |
+ (((reg_t) bytes[1]) << 8) |
+ (((reg_t) bytes[2]) << 16) |
+ (((reg_t) bytes[3]) << 24);
+ case 8:
+ return bytes[0] |
+ (((reg_t) bytes[1]) << 8) |
+ (((reg_t) bytes[2]) << 16) |
+ (((reg_t) bytes[3]) << 24) |
+ (((reg_t) bytes[4]) << 32) |
+ (((reg_t) bytes[5]) << 40) |
+ (((reg_t) bytes[6]) << 48) |
+ (((reg_t) bytes[7]) << 56);
+ }
+ abort();
+}
- if (pgbase == reg_t(-1)) {
- if (fetch) throw trap_instruction_access_fault(addr);
- else if (store) throw trap_store_access_fault(addr);
- else throw trap_load_access_fault(addr);
+void mmu_t::load_slow_path(reg_t addr, reg_t len, uint8_t* bytes)
+{
+ reg_t paddr = translate(addr, LOAD);
+
+ if (auto host_addr = sim->addr_to_mem(paddr)) {
+ memcpy(bytes, host_addr, len);
+ if (tracer.interested_in_range(paddr, paddr + PGSIZE, LOAD))
+ tracer.trace(paddr, len, LOAD);
+ else
+ refill_tlb(addr, paddr, host_addr, LOAD);
+ } else if (!sim->mmio_load(paddr, len, bytes)) {
+ throw trap_load_access_fault(addr);
}
- bool trace = tracer.interested_in_range(pgbase, pgbase + PGSIZE, store, fetch);
- if (unlikely(!fetch && trace))
- tracer.trace(paddr, bytes, store, fetch);
- else
- {
- if (tlb_load_tag[idx] != expected_tag) tlb_load_tag[idx] = -1;
- if (tlb_store_tag[idx] != expected_tag) tlb_store_tag[idx] = -1;
- if (tlb_insn_tag[idx] != expected_tag) tlb_insn_tag[idx] = -1;
+ if (!matched_trigger) {
+ reg_t data = reg_from_bytes(len, bytes);
+ matched_trigger = trigger_exception(OPERATION_LOAD, addr, data);
+ if (matched_trigger)
+ throw *matched_trigger;
+ }
+}
- if (fetch) tlb_insn_tag[idx] = expected_tag;
- else if (store) tlb_store_tag[idx] = expected_tag;
- else tlb_load_tag[idx] = expected_tag;
+void mmu_t::store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes)
+{
+ reg_t paddr = translate(addr, STORE);
- tlb_data[idx] = mem + pgbase - (addr & ~(PGSIZE-1));
+ if (!matched_trigger) {
+ reg_t data = reg_from_bytes(len, bytes);
+ matched_trigger = trigger_exception(OPERATION_STORE, addr, data);
+ if (matched_trigger)
+ throw *matched_trigger;
}
- return mem + paddr;
+ if (auto host_addr = sim->addr_to_mem(paddr)) {
+ memcpy(host_addr, bytes, len);
+ if (tracer.interested_in_range(paddr, paddr + PGSIZE, STORE))
+ tracer.trace(paddr, len, STORE);
+ else
+ refill_tlb(addr, paddr, host_addr, STORE);
+ } else if (!sim->mmio_store(paddr, len, bytes)) {
+ throw trap_store_access_fault(addr);
+ }
}
-reg_t mmu_t::walk(reg_t addr, bool supervisor, bool store, bool fetch)
+tlb_entry_t mmu_t::refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type)
{
- reg_t msb_mask = -(reg_t(1) << (VA_BITS(proc->xlen) - 1));
- if ((addr & msb_mask) != 0 && (addr & msb_mask) != msb_mask)
- return -1; // address isn't properly sign-extended
-
- reg_t base = proc->get_state()->sptbr;
+ reg_t idx = (vaddr >> PGSHIFT) % TLB_ENTRIES;
+ reg_t expected_tag = vaddr >> PGSHIFT;
+
+ if ((tlb_load_tag[idx] & ~TLB_CHECK_TRIGGERS) != expected_tag)
+ tlb_load_tag[idx] = -1;
+ if ((tlb_store_tag[idx] & ~TLB_CHECK_TRIGGERS) != expected_tag)
+ tlb_store_tag[idx] = -1;
+ if ((tlb_insn_tag[idx] & ~TLB_CHECK_TRIGGERS) != expected_tag)
+ tlb_insn_tag[idx] = -1;
+
+ if ((check_triggers_fetch && type == FETCH) ||
+ (check_triggers_load && type == LOAD) ||
+ (check_triggers_store && type == STORE))
+ expected_tag |= TLB_CHECK_TRIGGERS;
+
+ if (type == FETCH) tlb_insn_tag[idx] = expected_tag;
+ else if (type == STORE) tlb_store_tag[idx] = expected_tag;
+ else tlb_load_tag[idx] = expected_tag;
+
+ tlb_entry_t entry = {host_addr - vaddr, paddr - vaddr};
+ tlb_data[idx] = entry;
+ return entry;
+}
- int xlen = proc->max_xlen;
- int ptshift = (LEVELS(xlen) - 1) * PTIDXBITS(xlen);
- for (reg_t i = 0; i < LEVELS(xlen); i++, ptshift -= PTIDXBITS(xlen)) {
- reg_t idx = (addr >> (PGSHIFT+ptshift)) & ((1<<PTIDXBITS(xlen))-1);
+reg_t mmu_t::walk(reg_t addr, access_type type, reg_t mode)
+{
+ vm_info vm = decode_vm_info(proc->max_xlen, mode, proc->get_state()->satp);
+ if (vm.levels == 0)
+ return addr & ((reg_t(2) << (proc->xlen-1))-1); // zero-extend from xlen
+
+ bool s_mode = mode == PRV_S;
+ bool sum = get_field(proc->state.mstatus, MSTATUS_SUM);
+ bool mxr = get_field(proc->state.mstatus, MSTATUS_MXR);
+
+ // verify bits xlen-1:va_bits-1 are all equal
+ int va_bits = PGSHIFT + vm.levels * vm.idxbits;
+ reg_t mask = (reg_t(1) << (proc->xlen - (va_bits-1))) - 1;
+ reg_t masked_msbs = (addr >> (va_bits-1)) & mask;
+ if (masked_msbs != 0 && masked_msbs != mask)
+ vm.levels = 0;
+
+ reg_t base = vm.ptbase;
+ for (int i = vm.levels - 1; i >= 0; i--) {
+ int ptshift = i * vm.idxbits;
+ reg_t idx = (addr >> (PGSHIFT + ptshift)) & ((1 << vm.idxbits) - 1);
// check that physical address of PTE is legal
- reg_t pte_addr = base + idx*sizeof(reg_t);
- if (pte_addr >= memsz)
- return -1;
+ auto ppte = sim->addr_to_mem(base + idx * vm.ptesize);
+ if (!ppte)
+ goto fail_access;
- reg_t* ppte = (reg_t*)(mem+pte_addr);
- reg_t ppn = *ppte >> PPN_SHIFT(xlen);
+ reg_t pte = vm.ptesize == 4 ? *(uint32_t*)ppte : *(uint64_t*)ppte;
+ reg_t ppn = pte >> PTE_PPN_SHIFT;
- if ((*ppte & PTE_TYPE) == PTE_TYPE_TABLE) { // next level of page table
+ if (PTE_TABLE(pte)) { // next level of page table
base = ppn << PGSHIFT;
+ } else if ((pte & PTE_U) ? s_mode && (type == FETCH || !sum) : !s_mode) {
+ break;
+ } else if (!(pte & PTE_V) || (!(pte & PTE_R) && (pte & PTE_W))) {
+ break;
+ } else if (type == FETCH ? !(pte & PTE_X) :
+ type == LOAD ? !(pte & PTE_R) && !(mxr && (pte & PTE_X)) :
+ !((pte & PTE_R) && (pte & PTE_W))) {
+ break;
+ } else if ((ppn & ((reg_t(1) << ptshift) - 1)) != 0) {
+ break;
} else {
- // we've found the PTE. check the permissions.
- if (!PTE_CHECK_PERM(*ppte, supervisor, store, fetch))
- return -1;
- // set referenced and possibly dirty bits.
- *ppte |= PTE_R;
- if (store)
- *ppte |= PTE_D;
+ reg_t ad = PTE_A | ((type == STORE) * PTE_D);
+#ifdef RISCV_ENABLE_DIRTY
+ // set accessed and possibly dirty bits.
+ *(uint32_t*)ppte |= ad;
+#else
+ // take exception if access or possibly dirty bit is not set.
+ if ((pte & ad) != ad)
+ break;
+#endif
// for superpage mappings, make a fake leaf PTE for the TLB's benefit.
reg_t vpn = addr >> PGSHIFT;
- reg_t addr = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT;
-
- // check that physical address is legal
- if (addr >= memsz)
- return -1;
-
- return addr;
+ reg_t value = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT;
+ return value;
}
}
- return -1;
+fail:
+ switch (type) {
+ case FETCH: throw trap_instruction_page_fault(addr);
+ case LOAD: throw trap_load_page_fault(addr);
+ case STORE: throw trap_store_page_fault(addr);
+ default: abort();
+ }
+
+fail_access:
+ switch (type) {
+ case FETCH: throw trap_instruction_access_fault(addr);
+ case LOAD: throw trap_load_access_fault(addr);
+ case STORE: throw trap_store_access_fault(addr);
+ default: abort();
+ }
}
void mmu_t::register_memtracer(memtracer_t* t)
projects / riscv-isa-sim.git / blobdiff