X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=riscv%2Fmmu.h;h=4380448f955f73ff4e51f956ad43164076153bec;hb=d6fcfdebf6a893bf37670fd67203d18653df4a0e;hp=8fe3043bb77d4bcafe3a289983c7b6c6a4a7f9b6;hpb=7baa7795ba6b878f1dc859f4d4ee239bb569c750;p=riscv-isa-sim.git diff --git a/riscv/mmu.h b/riscv/mmu.h index 8fe3043..4380448 100644 --- a/riscv/mmu.h +++ b/riscv/mmu.h @@ -7,7 +7,7 @@ #include "trap.h" #include "common.h" #include "config.h" -#include "sim.h" +#include "simif.h" #include "processor.h" #include "memtracer.h" #include @@ -26,26 +26,75 @@ struct insn_fetch_t struct icache_entry_t { reg_t tag; - reg_t pad; + struct icache_entry_t* next; insn_fetch_t data; }; +struct tlb_entry_t { + char* host_offset; + reg_t target_offset; +}; + +class trigger_matched_t +{ + public: + trigger_matched_t(int index, + trigger_operation_t operation, reg_t address, reg_t data) : + index(index), operation(operation), address(address), data(data) {} + + int index; + trigger_operation_t operation; + reg_t address; + reg_t data; +}; + // this class implements a processor's port into the virtual memory system. // an MMU and instruction cache are maintained for simulator performance. class mmu_t { public: - mmu_t(sim_t* sim, processor_t* proc); + mmu_t(simif_t* sim, processor_t* proc); ~mmu_t(); + inline reg_t misaligned_load(reg_t addr, size_t size) + { +#ifdef RISCV_ENABLE_MISALIGNED + reg_t res = 0; + for (size_t i = 0; i < size; i++) + res += (reg_t)load_uint8(addr + i) << (i * 8); + return res; +#else + throw trap_load_address_misaligned(addr); +#endif + } + + inline void misaligned_store(reg_t addr, reg_t data, size_t size) + { +#ifdef RISCV_ENABLE_MISALIGNED + for (size_t i = 0; i < size; i++) + store_uint8(addr + i, data >> (i * 8)); +#else + throw trap_store_address_misaligned(addr); +#endif + } + // template for functions that load an aligned value from memory #define load_func(type) \ - type##_t load_##type(reg_t addr) __attribute__((always_inline)) { \ - if (addr & (sizeof(type##_t)-1)) \ - throw trap_load_address_misaligned(addr); \ + inline type##_t load_##type(reg_t addr) { \ + if (unlikely(addr & (sizeof(type##_t)-1))) \ + return misaligned_load(addr, sizeof(type##_t)); \ reg_t vpn = addr >> PGSHIFT; \ if (likely(tlb_load_tag[vpn % TLB_ENTRIES] == vpn)) \ - return *(type##_t*)(tlb_data[vpn % TLB_ENTRIES] + addr); \ + return *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \ + if (unlikely(tlb_load_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \ + type##_t data = *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \ + if (!matched_trigger) { \ + matched_trigger = trigger_exception(OPERATION_LOAD, addr, data); \ + if (matched_trigger) \ + throw *matched_trigger; \ + } \ + return data; \ + } \ type##_t res; \ load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res); \ return res; \ @@ -66,21 +115,71 @@ public: // template for functions that store an aligned value to memory #define store_func(type) \ void store_##type(reg_t addr, type##_t val) { \ - if (addr & (sizeof(type##_t)-1)) \ - throw trap_store_address_misaligned(addr); \ + if (unlikely(addr & (sizeof(type##_t)-1))) \ + return misaligned_store(addr, val, sizeof(type##_t)); \ reg_t vpn = addr >> PGSHIFT; \ if (likely(tlb_store_tag[vpn % TLB_ENTRIES] == vpn)) \ - *(type##_t*)(tlb_data[vpn % TLB_ENTRIES] + addr) = val; \ + *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \ + else if (unlikely(tlb_store_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \ + if (!matched_trigger) { \ + matched_trigger = trigger_exception(OPERATION_STORE, addr, val); \ + if (matched_trigger) \ + throw *matched_trigger; \ + } \ + *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \ + } \ else \ store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&val); \ } + // template for functions that perform an atomic memory operation + #define amo_func(type) \ + template \ + type##_t amo_##type(reg_t addr, op f) { \ + if (addr & (sizeof(type##_t)-1)) \ + throw trap_store_address_misaligned(addr); \ + try { \ + auto lhs = load_##type(addr); \ + store_##type(addr, f(lhs)); \ + return lhs; \ + } catch (trap_load_page_fault& t) { \ + /* AMO faults should be reported as store faults */ \ + throw trap_store_page_fault(t.get_tval()); \ + } catch (trap_load_access_fault& t) { \ + /* AMO faults should be reported as store faults */ \ + throw trap_store_access_fault(t.get_tval()); \ + } \ + } + + void store_float128(reg_t addr, float128_t val) + { +#ifndef RISCV_ENABLE_MISALIGNED + if (unlikely(addr & (sizeof(float128_t)-1))) + throw trap_store_address_misaligned(addr); +#endif + store_uint64(addr, val.v[0]); + store_uint64(addr + 8, val.v[1]); + } + + float128_t load_float128(reg_t addr) + { +#ifndef RISCV_ENABLE_MISALIGNED + if (unlikely(addr & (sizeof(float128_t)-1))) + throw trap_load_address_misaligned(addr); +#endif + return (float128_t){load_uint64(addr), load_uint64(addr + 8)}; + } + // store value to memory at aligned address store_func(uint8) store_func(uint16) store_func(uint32) store_func(uint64) + // perform an atomic memory operation at an aligned address + amo_func(uint32) + amo_func(uint64) + static const reg_t ICACHE_ENTRIES = 1024; inline size_t icache_index(reg_t addr) @@ -90,34 +189,34 @@ public: inline icache_entry_t* refill_icache(reg_t addr, icache_entry_t* entry) { - const uint16_t* iaddr = translate_insn_addr(addr); - insn_bits_t insn = *iaddr; + auto tlb_entry = translate_insn_addr(addr); + insn_bits_t insn = *(uint16_t*)(tlb_entry.host_offset + addr); int length = insn_length(insn); if (likely(length == 4)) { - insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr(addr + 2) << 16; + insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 2) << 16; } else if (length == 2) { insn = (int16_t)insn; } else if (length == 6) { - insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr(addr + 4) << 32; - insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr(addr + 2) << 16; + insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 4) << 32; + insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16; } else { static_assert(sizeof(insn_bits_t) == 8, "insn_bits_t must be uint64_t"); - insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr(addr + 6) << 48; - insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr(addr + 4) << 32; - insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr(addr + 2) << 16; + insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 6) << 48; + insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 4) << 32; + insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16; } insn_fetch_t fetch = {proc->decode_insn(insn), insn}; entry->tag = addr; + entry->next = &icache[icache_index(addr + length)]; entry->data = fetch; - reg_t paddr = sim->mem_to_addr((char*)iaddr); + reg_t paddr = tlb_entry.target_offset + addr;; if (tracer.interested_in_range(paddr, paddr + 1, FETCH)) { entry->tag = -1; tracer.trace(paddr, length, FETCH); } - entry->tag = -1; // TODO: this is hack to work around Debug RAM code being cached return entry; } @@ -131,7 +230,8 @@ public: inline insn_fetch_t load_insn(reg_t addr) { - return access_icache(addr)->data; + icache_entry_t entry; + return refill_icache(addr, &entry)->data; } void flush_tlb(); @@ -139,8 +239,26 @@ public: void register_memtracer(memtracer_t*); + int is_dirty_enabled() + { +#ifdef RISCV_ENABLE_DIRTY + return 1; +#else + return 0; +#endif + } + + int is_misaligned_enabled() + { +#ifdef RISCV_ENABLE_MISALIGNED + return 1; +#else + return 0; +#endif + } + private: - sim_t* sim; + simif_t* sim; processor_t* proc; memtracer_list_t tracer; uint16_t fetch_temp; @@ -150,33 +268,99 @@ private: // implement a TLB for simulator performance static const reg_t TLB_ENTRIES = 256; - char* tlb_data[TLB_ENTRIES]; + // If a TLB tag has TLB_CHECK_TRIGGERS set, then the MMU must check for a + // trigger match before completing an access. + static const reg_t TLB_CHECK_TRIGGERS = reg_t(1) << 63; + tlb_entry_t tlb_data[TLB_ENTRIES]; reg_t tlb_insn_tag[TLB_ENTRIES]; reg_t tlb_load_tag[TLB_ENTRIES]; reg_t tlb_store_tag[TLB_ENTRIES]; // finish translation on a TLB miss and update the TLB - void refill_tlb(reg_t vaddr, reg_t paddr, access_type type); + tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type); const char* fill_from_mmio(reg_t vaddr, reg_t paddr); // perform a page table walk for a given VA; set referenced/dirty bits - reg_t walk(reg_t addr, access_type type, bool supervisor, bool pum); + reg_t walk(reg_t addr, access_type type, reg_t prv); // handle uncommon cases: TLB misses, page faults, MMIO - const uint16_t* fetch_slow_path(reg_t addr); + tlb_entry_t fetch_slow_path(reg_t addr); void load_slow_path(reg_t addr, reg_t len, uint8_t* bytes); void store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes); reg_t translate(reg_t addr, access_type type); // ITLB lookup - const uint16_t* translate_insn_addr(reg_t addr) __attribute__((always_inline)) { + inline tlb_entry_t translate_insn_addr(reg_t addr) { reg_t vpn = addr >> PGSHIFT; if (likely(tlb_insn_tag[vpn % TLB_ENTRIES] == vpn)) - return (uint16_t*)(tlb_data[vpn % TLB_ENTRIES] + addr); + return tlb_data[vpn % TLB_ENTRIES]; + if (unlikely(tlb_insn_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { + uint16_t* ptr = (uint16_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); + int match = proc->trigger_match(OPERATION_EXECUTE, addr, *ptr); + if (match >= 0) + throw trigger_matched_t(match, OPERATION_EXECUTE, addr, *ptr); + return tlb_data[vpn % TLB_ENTRIES]; + } return fetch_slow_path(addr); } + inline const uint16_t* translate_insn_addr_to_host(reg_t addr) { + return (uint16_t*)(translate_insn_addr(addr).host_offset + addr); + } + + inline trigger_matched_t *trigger_exception(trigger_operation_t operation, + reg_t address, reg_t data) + { + if (!proc) { + return NULL; + } + int match = proc->trigger_match(operation, address, data); + if (match == -1) + return NULL; + if (proc->state.mcontrol[match].timing == 0) { + throw trigger_matched_t(match, operation, address, data); + } + return new trigger_matched_t(match, operation, address, data); + } + + bool check_triggers_fetch; + bool check_triggers_load; + bool check_triggers_store; + // The exception describing a matched trigger, or NULL. + trigger_matched_t *matched_trigger; + friend class processor_t; }; +struct vm_info { + int levels; + int idxbits; + int ptesize; + reg_t ptbase; +}; + +inline vm_info decode_vm_info(int xlen, reg_t prv, reg_t satp) +{ + if (prv == PRV_M) { + return {0, 0, 0, 0}; + } else if (prv <= PRV_S && xlen == 32) { + switch (get_field(satp, SATP32_MODE)) { + case SATP_MODE_OFF: return {0, 0, 0, 0}; + case SATP_MODE_SV32: return {2, 10, 4, (satp & SATP32_PPN) << PGSHIFT}; + default: abort(); + } + } else if (prv <= PRV_S && xlen == 64) { + switch (get_field(satp, SATP64_MODE)) { + case SATP_MODE_OFF: return {0, 0, 0, 0}; + case SATP_MODE_SV39: return {3, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_SV48: return {4, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_SV57: return {5, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; + case SATP_MODE_SV64: return {6, 9, 8, (satp & SATP64_PPN) << PGSHIFT}; + default: abort(); + } + } else { + abort(); + } +} + #endif