+// See LICENSE for license details.
+
+#ifndef _RISCV_MMU_H
+#define _RISCV_MMU_H
+
#include "decode.h"
#include "trap.h"
-#include "icsim.h"
#include "common.h"
-#include <assert.h>
+#include "config.h"
+#include "sim.h"
+#include "processor.h"
+#include "memtracer.h"
+#include <stdlib.h>
+#include <vector>
+
+// virtual memory configuration
+#define PGSHIFT 12
+const reg_t PGSIZE = 1 << PGSHIFT;
+const reg_t PGMASK = ~(PGSIZE-1);
+
+struct insn_fetch_t
+{
+ insn_func_t func;
+ insn_t insn;
+};
-class processor_t;
+struct icache_entry_t {
+ reg_t tag;
+ struct icache_entry_t* next;
+ insn_fetch_t data;
+};
-typedef reg_t pte_t;
+struct tlb_entry_t {
+ char* host_offset;
+ reg_t target_offset;
+};
-const reg_t LEVELS = 4;
-const reg_t PGSHIFT = 12;
-const reg_t PGSIZE = 1 << PGSHIFT;
-const reg_t PTIDXBITS = PGSHIFT - (sizeof(pte_t) == 8 ? 3 : 2);
-const reg_t PPN_BITS = 8*sizeof(reg_t) - PGSHIFT;
-
-#define PTE_T 0x001 // Entry is a page Table descriptor
-#define PTE_E 0x002 // Entry is a page table Entry
-#define PTE_R 0x004 // Referenced
-#define PTE_D 0x008 // Dirty
-#define PTE_UX 0x010 // User eXecute permission
-#define PTE_UW 0x020 // User Read permission
-#define PTE_UR 0x040 // User Write permission
-#define PTE_SX 0x080 // Supervisor eXecute permission
-#define PTE_SW 0x100 // Supervisor Read permission
-#define PTE_SR 0x200 // Supervisor Write permission
-#define PTE_PERM (PTE_SR | PTE_SW | PTE_SX | PTE_UR | PTE_UW | PTE_UX)
-#define PTE_PERM_SHIFT 4
-#define PTE_PPN_SHIFT 12
+class trigger_matched_t
+{
+ public:
+ trigger_matched_t(int index,
+ trigger_operation_t operation, reg_t address, reg_t data) :
+ index(index), operation(operation), address(address), data(data) {}
+
+ int index;
+ trigger_operation_t operation;
+ reg_t address;
+ reg_t data;
+};
+// this class implements a processor's port into the virtual memory system.
+// an MMU and instruction cache are maintained for simulator performance.
class mmu_t
{
public:
- mmu_t(char* _mem, size_t _memsz)
- : mem(_mem), memsz(_memsz), badvaddr(0),
- ptbr(0), supervisor(true), vm_enabled(false),
- icsim(NULL), dcsim(NULL), itlbsim(NULL), dtlbsim(NULL)
+ mmu_t(simif_t* sim, processor_t* proc);
+ ~mmu_t();
+
+ inline reg_t misaligned_load(reg_t addr, size_t size)
{
+#ifdef RISCV_ENABLE_MISALIGNED
+ reg_t res = 0;
+ for (size_t i = 0; i < size; i++)
+ res += (reg_t)load_uint8(addr + i) << (i * 8);
+ return res;
+#else
+ throw trap_load_address_misaligned(addr);
+#endif
}
- #ifdef RISCV_ENABLE_ICSIM
- # define dcsim_tick(dcsim, dtlbsim, addr, size, st) \
- do { if(dcsim) (dcsim)->tick(addr, size, st); \
- if(dtlbsim) (dtlbsim)->tick(addr, sizeof(reg_t), false); } while(0)
- #else
- # define dcsim_tick(dcsim, dtlbsim, addr, size, st)
- #endif
+ inline void misaligned_store(reg_t addr, reg_t data, size_t size)
+ {
+#ifdef RISCV_ENABLE_MISALIGNED
+ for (size_t i = 0; i < size; i++)
+ store_uint8(addr + i, data >> (i * 8));
+#else
+ throw trap_store_address_misaligned(addr);
+#endif
+ }
+ // template for functions that load an aligned value from memory
#define load_func(type) \
- type##_t load_##type(reg_t addr) { \
- if(unlikely(addr % sizeof(type##_t))) \
- { \
- badvaddr = addr; \
- throw trap_load_address_misaligned; \
- } \
- addr = translate(addr, false, false); \
- dcsim_tick(dcsim, dtlbsim, addr, sizeof(type##_t), false); \
- return *(type##_t*)(mem+addr); \
- }
-
- #define store_func(type) \
- void store_##type(reg_t addr, type##_t val) { \
- if(unlikely(addr % sizeof(type##_t))) \
- { \
- badvaddr = addr; \
- throw trap_store_address_misaligned; \
+ inline type##_t load_##type(reg_t addr) { \
+ if (unlikely(addr & (sizeof(type##_t)-1))) \
+ return misaligned_load(addr, sizeof(type##_t)); \
+ reg_t vpn = addr >> PGSHIFT; \
+ if (likely(tlb_load_tag[vpn % TLB_ENTRIES] == vpn)) \
+ return *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
+ if (unlikely(tlb_load_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
+ type##_t data = *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
+ if (!matched_trigger) { \
+ matched_trigger = trigger_exception(OPERATION_LOAD, addr, data); \
+ if (matched_trigger) \
+ throw *matched_trigger; \
+ } \
+ return data; \
} \
- addr = translate(addr, true, false); \
- dcsim_tick(dcsim, dtlbsim, addr, sizeof(type##_t), true); \
- *(type##_t*)(mem+addr) = val; \
+ type##_t res; \
+ load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res); \
+ return res; \
}
- insn_t __attribute__((always_inline)) load_insn(reg_t addr, bool rvc)
- {
- insn_t insn;
-
- reg_t idx = (addr/sizeof(insn_t)) % ICACHE_ENTRIES;
- bool hit = addr % 4 == 0 && icache_tag[idx] == (addr | 1);
- if(likely(hit))
- return icache_data[idx];
-
- #ifdef RISCV_ENABLE_RVC
- if(addr % 4 == 2 && rvc)
- {
- reg_t paddr_lo = translate(addr, false, true);
- insn.bits = *(uint16_t*)(mem+paddr_lo);
-
- if(!INSN_IS_RVC(insn.bits))
- {
- reg_t paddr_hi = translate(addr+2, false, true);
- insn.bits |= (uint32_t)*(uint16_t*)(mem+paddr_hi) << 16;
- }
- }
- else
- #endif
- {
- if(unlikely(addr % 4))
- {
- badvaddr = addr;
- throw trap_instruction_address_misaligned;
- }
- reg_t paddr = translate(addr, false, true);
- insn = *(insn_t*)(mem+paddr);
-
- icache_tag[idx] = addr | 1;
- icache_data[idx] = insn;
- }
-
- #ifdef RISCV_ENABLE_ICSIM
- if(icsim)
- icsim->tick(addr, insn_length(insn.bits), false);
- if(itlbsim)
- itlbsim->tick(addr, sizeof(reg_t), false);
- #endif
-
- return insn;
- }
-
+ // load value from memory at aligned address; zero extend to register width
load_func(uint8)
load_func(uint16)
load_func(uint32)
load_func(uint64)
+ // load value from memory at aligned address; sign extend to register width
load_func(int8)
load_func(int16)
load_func(int32)
load_func(int64)
+ // template for functions that store an aligned value to memory
+ #define store_func(type) \
+ void store_##type(reg_t addr, type##_t val) { \
+ if (unlikely(addr & (sizeof(type##_t)-1))) \
+ return misaligned_store(addr, val, sizeof(type##_t)); \
+ reg_t vpn = addr >> PGSHIFT; \
+ if (likely(tlb_store_tag[vpn % TLB_ENTRIES] == vpn)) \
+ *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
+ else if (unlikely(tlb_store_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
+ if (!matched_trigger) { \
+ matched_trigger = trigger_exception(OPERATION_STORE, addr, val); \
+ if (matched_trigger) \
+ throw *matched_trigger; \
+ } \
+ *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
+ } \
+ else \
+ store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&val); \
+ }
+
+ // template for functions that perform an atomic memory operation
+ #define amo_func(type) \
+ template<typename op> \
+ type##_t amo_##type(reg_t addr, op f) { \
+ if (addr & (sizeof(type##_t)-1)) \
+ throw trap_store_address_misaligned(addr); \
+ try { \
+ auto lhs = load_##type(addr); \
+ store_##type(addr, f(lhs)); \
+ return lhs; \
+ } catch (trap_load_page_fault& t) { \
+ /* AMO faults should be reported as store faults */ \
+ throw trap_store_page_fault(t.get_tval()); \
+ } catch (trap_load_access_fault& t) { \
+ /* AMO faults should be reported as store faults */ \
+ throw trap_store_access_fault(t.get_tval()); \
+ } \
+ }
+
+ void store_float128(reg_t addr, float128_t val)
+ {
+#ifndef RISCV_ENABLE_MISALIGNED
+ if (unlikely(addr & (sizeof(float128_t)-1)))
+ throw trap_store_address_misaligned(addr);
+#endif
+ store_uint64(addr, val.v[0]);
+ store_uint64(addr + 8, val.v[1]);
+ }
+
+ float128_t load_float128(reg_t addr)
+ {
+#ifndef RISCV_ENABLE_MISALIGNED
+ if (unlikely(addr & (sizeof(float128_t)-1)))
+ throw trap_load_address_misaligned(addr);
+#endif
+ return (float128_t){load_uint64(addr), load_uint64(addr + 8)};
+ }
+
+ // store value to memory at aligned address
store_func(uint8)
store_func(uint16)
store_func(uint32)
store_func(uint64)
- reg_t get_badvaddr() { return badvaddr; }
- reg_t get_ptbr() { return ptbr; }
+ // perform an atomic memory operation at an aligned address
+ amo_func(uint32)
+ amo_func(uint64)
- void set_supervisor(bool sup) { supervisor = sup; }
- void set_vm_enabled(bool en) { vm_enabled = en; }
- void set_ptbr(reg_t addr) { ptbr = addr & ~(PGSIZE-1); flush_tlb(); }
+ static const reg_t ICACHE_ENTRIES = 1024;
- void set_icsim(icsim_t* _icsim) { icsim = _icsim; }
- void set_dcsim(icsim_t* _dcsim) { dcsim = _dcsim; }
- void set_itlbsim(icsim_t* _itlbsim) { itlbsim = _itlbsim; }
- void set_dtlbsim(icsim_t* _dtlbsim) { dtlbsim = _dtlbsim; }
+ inline size_t icache_index(reg_t addr)
+ {
+ return (addr / PC_ALIGN) % ICACHE_ENTRIES;
+ }
- void flush_tlb();
- void flush_icache();
+ inline icache_entry_t* refill_icache(reg_t addr, icache_entry_t* entry)
+ {
+ auto tlb_entry = translate_insn_addr(addr);
+ insn_bits_t insn = *(uint16_t*)(tlb_entry.host_offset + addr);
+ int length = insn_length(insn);
+
+ if (likely(length == 4)) {
+ insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 2) << 16;
+ } else if (length == 2) {
+ insn = (int16_t)insn;
+ } else if (length == 6) {
+ insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 4) << 32;
+ insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16;
+ } else {
+ static_assert(sizeof(insn_bits_t) == 8, "insn_bits_t must be uint64_t");
+ insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 6) << 48;
+ insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 4) << 32;
+ insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16;
+ }
-private:
- char* mem;
- size_t memsz;
- reg_t badvaddr;
+ insn_fetch_t fetch = {proc->decode_insn(insn), insn};
+ entry->tag = addr;
+ entry->next = &icache[icache_index(addr + length)];
+ entry->data = fetch;
- reg_t ptbr;
- bool supervisor;
- bool vm_enabled;
+ reg_t paddr = tlb_entry.target_offset + addr;;
+ if (tracer.interested_in_range(paddr, paddr + 1, FETCH)) {
+ entry->tag = -1;
+ tracer.trace(paddr, length, FETCH);
+ }
+ return entry;
+ }
- static const reg_t TLB_ENTRIES = 256;
- pte_t tlb_data[TLB_ENTRIES];
- reg_t tlb_tag[TLB_ENTRIES];
+ inline icache_entry_t* access_icache(reg_t addr)
+ {
+ icache_entry_t* entry = &icache[icache_index(addr)];
+ if (likely(entry->tag == addr))
+ return entry;
+ return refill_icache(addr, entry);
+ }
- static const reg_t ICACHE_ENTRIES = 256;
- insn_t icache_data[ICACHE_ENTRIES];
- reg_t icache_tag[ICACHE_ENTRIES];
+ inline insn_fetch_t load_insn(reg_t addr)
+ {
+ icache_entry_t entry;
+ return refill_icache(addr, &entry)->data;
+ }
- icsim_t* icsim;
- icsim_t* dcsim;
- icsim_t* itlbsim;
- icsim_t* dtlbsim;
+ void flush_tlb();
+ void flush_icache();
- reg_t translate(reg_t addr, bool store, bool fetch)
+ void register_memtracer(memtracer_t*);
+
+ int is_dirty_enabled()
{
- reg_t idx = (addr >> PGSHIFT) % TLB_ENTRIES;
- pte_t pte = tlb_data[idx];
- reg_t tag = tlb_tag[idx];
-
- trap_t trap = store ? trap_store_access_fault
- : fetch ? trap_instruction_access_fault
- : trap_load_access_fault;
-
- bool hit = (pte & PTE_E) && tag == (addr >> PGSHIFT);
- if(unlikely(!hit))
- {
- pte = walk(addr);
- if(!(pte & PTE_E))
- {
- badvaddr = addr;
- throw trap;
- }
-
- tlb_data[idx] = pte;
- tlb_tag[idx] = addr >> PGSHIFT;
- }
+#ifdef RISCV_ENABLE_DIRTY
+ return 1;
+#else
+ return 0;
+#endif
+ }
+
+ int is_misaligned_enabled()
+ {
+#ifdef RISCV_ENABLE_MISALIGNED
+ return 1;
+#else
+ return 0;
+#endif
+ }
+
+private:
+ simif_t* sim;
+ processor_t* proc;
+ memtracer_list_t tracer;
+ uint16_t fetch_temp;
+
+ // implement an instruction cache for simulator performance
+ icache_entry_t icache[ICACHE_ENTRIES];
- reg_t access_type = store ? PTE_UW : fetch ? PTE_UX : PTE_UR;
- if(supervisor)
- access_type <<= 3;
- if(unlikely(!(access_type & pte & PTE_PERM)))
- {
- badvaddr = addr;
- throw trap;
+ // implement a TLB for simulator performance
+ static const reg_t TLB_ENTRIES = 256;
+ // If a TLB tag has TLB_CHECK_TRIGGERS set, then the MMU must check for a
+ // trigger match before completing an access.
+ static const reg_t TLB_CHECK_TRIGGERS = reg_t(1) << 63;
+ tlb_entry_t tlb_data[TLB_ENTRIES];
+ reg_t tlb_insn_tag[TLB_ENTRIES];
+ reg_t tlb_load_tag[TLB_ENTRIES];
+ reg_t tlb_store_tag[TLB_ENTRIES];
+
+ // finish translation on a TLB miss and update the TLB
+ tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type);
+ const char* fill_from_mmio(reg_t vaddr, reg_t paddr);
+
+ // perform a page table walk for a given VA; set referenced/dirty bits
+ reg_t walk(reg_t addr, access_type type, reg_t prv);
+
+ // handle uncommon cases: TLB misses, page faults, MMIO
+ tlb_entry_t fetch_slow_path(reg_t addr);
+ void load_slow_path(reg_t addr, reg_t len, uint8_t* bytes);
+ void store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes);
+ reg_t translate(reg_t addr, access_type type);
+
+ // ITLB lookup
+ inline tlb_entry_t translate_insn_addr(reg_t addr) {
+ reg_t vpn = addr >> PGSHIFT;
+ if (likely(tlb_insn_tag[vpn % TLB_ENTRIES] == vpn))
+ return tlb_data[vpn % TLB_ENTRIES];
+ if (unlikely(tlb_insn_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) {
+ uint16_t* ptr = (uint16_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr);
+ int match = proc->trigger_match(OPERATION_EXECUTE, addr, *ptr);
+ if (match >= 0)
+ throw trigger_matched_t(match, OPERATION_EXECUTE, addr, *ptr);
+ return tlb_data[vpn % TLB_ENTRIES];
}
+ return fetch_slow_path(addr);
+ }
- return (addr & (PGSIZE-1)) | ((pte >> PTE_PPN_SHIFT) << PGSHIFT);
+ inline const uint16_t* translate_insn_addr_to_host(reg_t addr) {
+ return (uint16_t*)(translate_insn_addr(addr).host_offset + addr);
}
- pte_t walk(reg_t addr)
+ inline trigger_matched_t *trigger_exception(trigger_operation_t operation,
+ reg_t address, reg_t data)
{
- pte_t pte = 0;
-
- if(!vm_enabled)
- {
- if(addr < memsz)
- pte = PTE_E | PTE_PERM | ((addr >> PGSHIFT) << PTE_PPN_SHIFT);
+ if (!proc) {
+ return NULL;
}
- else
- {
- reg_t base = ptbr;
- reg_t ptd;
-
- int ptshift = (LEVELS-1)*PTIDXBITS;
- for(reg_t i = 0; i < LEVELS; i++, ptshift -= PTIDXBITS)
- {
- reg_t idx = (addr >> (PGSHIFT+ptshift)) & ((1<<PTIDXBITS)-1);
-
- reg_t pte_addr = base + idx*sizeof(pte_t);
- if(pte_addr >= memsz)
- break;
-
- ptd = *(pte_t*)(mem+pte_addr);
- if(ptd & PTE_E)
- {
- // if this PTE is from a larger PT, fake a leaf
- // PTE so the TLB will work right
- reg_t vpn = addr >> PGSHIFT;
- ptd |= (vpn & ((1<<(ptshift))-1)) << PTE_PPN_SHIFT;
-
- // fault if physical addr is invalid
- reg_t ppn = ptd >> PTE_PPN_SHIFT;
- if((ppn << PGSHIFT) + (addr & (PGSIZE-1)) < memsz)
- pte = ptd;
- break;
- }
- else if(!(ptd & PTE_T))
- break;
-
- base = (ptd >> PTE_PPN_SHIFT) << PGSHIFT;
- }
+ int match = proc->trigger_match(operation, address, data);
+ if (match == -1)
+ return NULL;
+ if (proc->state.mcontrol[match].timing == 0) {
+ throw trigger_matched_t(match, operation, address, data);
}
-
- return pte;
+ return new trigger_matched_t(match, operation, address, data);
}
-
+
+ bool check_triggers_fetch;
+ bool check_triggers_load;
+ bool check_triggers_store;
+ // The exception describing a matched trigger, or NULL.
+ trigger_matched_t *matched_trigger;
+
friend class processor_t;
};
+
+struct vm_info {
+ int levels;
+ int idxbits;
+ int ptesize;
+ reg_t ptbase;
+};
+
+inline vm_info decode_vm_info(int xlen, reg_t prv, reg_t satp)
+{
+ if (prv == PRV_M) {
+ return {0, 0, 0, 0};
+ } else if (prv <= PRV_S && xlen == 32) {
+ switch (get_field(satp, SATP32_MODE)) {
+ case SATP_MODE_OFF: return {0, 0, 0, 0};
+ case SATP_MODE_SV32: return {2, 10, 4, (satp & SATP32_PPN) << PGSHIFT};
+ default: abort();
+ }
+ } else if (prv <= PRV_S && xlen == 64) {
+ switch (get_field(satp, SATP64_MODE)) {
+ case SATP_MODE_OFF: return {0, 0, 0, 0};
+ case SATP_MODE_SV39: return {3, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+ case SATP_MODE_SV48: return {4, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+ case SATP_MODE_SV57: return {5, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+ case SATP_MODE_SV64: return {6, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+ default: abort();
+ }
+ } else {
+ abort();
+ }
+}
+
+#endif
projects / riscv-isa-sim.git / blobdiff