X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=riscv%2Fmmu.h;h=a3f06c60e9d920ea81f78ee7c284ea4e46085fcf;hb=2dbcb01ca1c026b867cf673203646d213f6e6b5c;hp=d9a1ce3540f7138f99b0d5a891822180678a7966;hpb=ba9e6314deefb00a13e5bb25575e4ccdb713faec;p=riscv-isa-sim.git

diff --git a/riscv/mmu.h b/riscv/mmu.h
index d9a1ce3..a3f06c6 100644
--- a/riscv/mmu.h
+++ b/riscv/mmu.h
@@ -1,53 +1,103 @@
+// See LICENSE for license details.
+
 #ifndef _RISCV_MMU_H
 #define _RISCV_MMU_H
 
 #include "decode.h"
 #include "trap.h"
 #include "common.h"
+#include "config.h"
+#include "sim.h"
 #include "processor.h"
-
-class processor_t;
+#include "memtracer.h"
+#include <stdlib.h>
+#include <vector>
 
 // virtual memory configuration
-typedef reg_t pte_t;
-const reg_t LEVELS = sizeof(pte_t) == sizeof(uint64_t) ? 3 : 2;
-const reg_t PGSHIFT = 13;
+#define PGSHIFT 12
 const reg_t PGSIZE = 1 << PGSHIFT;
-const reg_t PTIDXBITS = PGSHIFT - (sizeof(pte_t) == 8 ? 3 : 2);
-const reg_t PPN_BITS = 8*sizeof(reg_t) - PGSHIFT;
-
-// page table entry (PTE) fields
-#define PTE_T    0x001 // Entry is a page Table descriptor
-#define PTE_E    0x002 // Entry is a page table Entry
-#define PTE_R    0x004 // Referenced
-#define PTE_D    0x008 // Dirty
-#define PTE_UX   0x010 // User eXecute permission
-#define PTE_UW   0x020 // User Read permission
-#define PTE_UR   0x040 // User Write permission
-#define PTE_SX   0x080 // Supervisor eXecute permission
-#define PTE_SW   0x100 // Supervisor Read permission
-#define PTE_SR   0x200 // Supervisor Write permission
-#define PTE_PERM (PTE_SR | PTE_SW | PTE_SX | PTE_UR | PTE_UW | PTE_UX)
-#define PTE_PPN_SHIFT  13 // LSB of physical page number in the PTE
+const reg_t PGMASK = ~(PGSIZE-1);
+
+struct insn_fetch_t
+{
+  insn_func_t func;
+  insn_t insn;
+};
+
+struct icache_entry_t {
+  reg_t tag;
+  struct icache_entry_t* next;
+  insn_fetch_t data;
+};
+
+struct tlb_entry_t {
+  char* host_offset;
+  reg_t target_offset;
+};
+
+class trigger_matched_t
+{
+  public:
+    trigger_matched_t(int index,
+        trigger_operation_t operation, reg_t address, reg_t data) :
+      index(index), operation(operation), address(address), data(data) {}
+
+    int index;
+    trigger_operation_t operation;
+    reg_t address;
+    reg_t data;
+};
 
 // this class implements a processor's port into the virtual memory system.
 // an MMU and instruction cache are maintained for simulator performance.
 class mmu_t
 {
 public:
-  mmu_t(char* _mem, size_t _memsz);
+  mmu_t(simif_t* sim, processor_t* proc);
   ~mmu_t();
 
+  inline reg_t misaligned_load(reg_t addr, size_t size)
+  {
+#ifdef RISCV_ENABLE_MISALIGNED
+    reg_t res = 0;
+    for (size_t i = 0; i < size; i++)
+      res += (reg_t)load_uint8(addr + i) << (i * 8);
+    return res;
+#else
+    throw trap_load_address_misaligned(addr);
+#endif
+  }
+
+  inline void misaligned_store(reg_t addr, reg_t data, size_t size)
+  {
+#ifdef RISCV_ENABLE_MISALIGNED
+    for (size_t i = 0; i < size; i++)
+      store_uint8(addr + i, data >> (i * 8));
+#else
+    throw trap_store_address_misaligned(addr);
+#endif
+  }
+
   // template for functions that load an aligned value from memory
   #define load_func(type) \
-    type##_t load_##type(reg_t addr) { \
-      if(unlikely(addr % sizeof(type##_t))) \
-      { \
-        badvaddr = addr; \
-        throw trap_load_address_misaligned; \
+    inline type##_t load_##type(reg_t addr) { \
+      if (unlikely(addr & (sizeof(type##_t)-1))) \
+        return misaligned_load(addr, sizeof(type##_t)); \
+      reg_t vpn = addr >> PGSHIFT; \
+      if (likely(tlb_load_tag[vpn % TLB_ENTRIES] == vpn)) \
+        return *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
+      if (unlikely(tlb_load_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
+        type##_t data = *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
+        if (!matched_trigger) { \
+          matched_trigger = trigger_exception(OPERATION_LOAD, addr, data); \
+          if (matched_trigger) \
+            throw *matched_trigger; \
+        } \
+        return data; \
       } \
-      void* paddr = translate(addr, false, false); \
-      return *(type##_t*)paddr; \
+      type##_t res; \
+      load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res); \
+      return res; \
     }
 
   // load value from memory at aligned address; zero extend to register width
@@ -65,126 +115,252 @@ public:
   // template for functions that store an aligned value to memory
   #define store_func(type) \
     void store_##type(reg_t addr, type##_t val) { \
-      if(unlikely(addr % sizeof(type##_t))) \
-      { \
-        badvaddr = addr; \
-        throw trap_store_address_misaligned; \
+      if (unlikely(addr & (sizeof(type##_t)-1))) \
+        return misaligned_store(addr, val, sizeof(type##_t)); \
+      reg_t vpn = addr >> PGSHIFT; \
+      if (likely(tlb_store_tag[vpn % TLB_ENTRIES] == vpn)) \
+        *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
+      else if (unlikely(tlb_store_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
+        if (!matched_trigger) { \
+          matched_trigger = trigger_exception(OPERATION_STORE, addr, val); \
+          if (matched_trigger) \
+            throw *matched_trigger; \
+        } \
+        *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
+      } \
+      else \
+        store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&val); \
+    }
+
+  // template for functions that perform an atomic memory operation
+  #define amo_func(type) \
+    template<typename op> \
+    type##_t amo_##type(reg_t addr, op f) { \
+      if (addr & (sizeof(type##_t)-1)) \
+        throw trap_store_address_misaligned(addr); \
+      try { \
+        auto lhs = load_##type(addr); \
+        store_##type(addr, f(lhs)); \
+        return lhs; \
+      } catch (trap_load_page_fault& t) { \
+        /* AMO faults should be reported as store faults */ \
+        throw trap_store_page_fault(t.get_tval()); \
+      } catch (trap_load_access_fault& t) { \
+        /* AMO faults should be reported as store faults */ \
+        throw trap_store_access_fault(t.get_tval()); \
       } \
-      void* paddr = translate(addr, true, false); \
-      *(type##_t*)paddr = val; \
     }
 
+  void store_float128(reg_t addr, float128_t val)
+  {
+#ifndef RISCV_ENABLE_MISALIGNED
+    if (unlikely(addr & (sizeof(float128_t)-1)))
+      throw trap_store_address_misaligned(addr);
+#endif
+    store_uint64(addr, val.v[0]);
+    store_uint64(addr + 8, val.v[1]);
+  }
+
+  float128_t load_float128(reg_t addr)
+  {
+#ifndef RISCV_ENABLE_MISALIGNED
+    if (unlikely(addr & (sizeof(float128_t)-1)))
+      throw trap_load_address_misaligned(addr);
+#endif
+    return (float128_t){load_uint64(addr), load_uint64(addr + 8)};
+  }
+
   // store value to memory at aligned address
   store_func(uint8)
   store_func(uint16)
   store_func(uint32)
   store_func(uint64)
 
-  // load instruction from memory at aligned address.
-  // (needed because instruction alignment requirement is variable
-  // if RVC is supported)
-  // returns the instruction at the specified address, given the current
-  // RVC mode.  func is set to a pointer to a function that knows how to
-  // execute the returned instruction.
-  insn_t __attribute__((always_inline)) load_insn(reg_t addr, bool rvc,
-                                                  insn_func_t* func)
+  // perform an atomic memory operation at an aligned address
+  amo_func(uint32)
+  amo_func(uint64)
+
+  static const reg_t ICACHE_ENTRIES = 1024;
+
+  inline size_t icache_index(reg_t addr)
   {
-    insn_t insn;
-
-    #ifdef RISCV_ENABLE_RVC
-    if(addr % 4 == 2 && rvc) // fetch across word boundary
-    {
-      void* addr_lo = translate(addr, false, true);
-      insn.bits = *(uint16_t*)addr_lo;
-
-      *func = processor_t::dispatch_table
-               [insn.bits % processor_t::DISPATCH_TABLE_SIZE];
-
-      if(!INSN_IS_RVC(insn.bits))
-      {
-        void* addr_hi = translate(addr+2, false, true);
-        insn.bits |= (uint32_t)*(uint16_t*)addr_hi << 16;
-      }
-    }
-    else
-    #endif
-    {
-      reg_t idx = (addr/sizeof(insn_t)) % ICACHE_ENTRIES;
-      insn_t data = icache_data[idx];
-      *func = icache_func[idx];
-      if(likely(icache_tag[idx] == addr))
-        return data;
-
-      // the processor guarantees alignment based upon rvc mode
-      void* paddr = translate(addr, false, true);
-      insn = *(insn_t*)paddr;
-
-      icache_tag[idx] = addr;
-      icache_data[idx] = insn;
-      icache_func[idx] = *func = processor_t::dispatch_table
-                                 [insn.bits % processor_t::DISPATCH_TABLE_SIZE];
+    return (addr / PC_ALIGN) % ICACHE_ENTRIES;
+  }
+
+  inline icache_entry_t* refill_icache(reg_t addr, icache_entry_t* entry)
+  {
+    auto tlb_entry = translate_insn_addr(addr);
+    insn_bits_t insn = *(uint16_t*)(tlb_entry.host_offset + addr);
+    int length = insn_length(insn);
+
+    if (likely(length == 4)) {
+      insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 2) << 16;
+    } else if (length == 2) {
+      insn = (int16_t)insn;
+    } else if (length == 6) {
+      insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 4) << 32;
+      insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16;
+    } else {
+      static_assert(sizeof(insn_bits_t) == 8, "insn_bits_t must be uint64_t");
+      insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 6) << 48;
+      insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 4) << 32;
+      insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16;
     }
 
-    return insn;
-  }
+    insn_fetch_t fetch = {proc->decode_insn(insn), insn};
+    entry->tag = addr;
+    entry->next = &icache[icache_index(addr + length)];
+    entry->data = fetch;
 
-  // get the virtual address that caused a fault
-  reg_t get_badvaddr() { return badvaddr; }
+    reg_t paddr = tlb_entry.target_offset + addr;;
+    if (tracer.interested_in_range(paddr, paddr + 1, FETCH)) {
+      entry->tag = -1;
+      tracer.trace(paddr, length, FETCH);
+    }
+    return entry;
+  }
 
-  // get/set the page table base register
-  reg_t get_ptbr() { return ptbr; }
-  void set_ptbr(reg_t addr) { ptbr = addr & ~(PGSIZE-1); flush_tlb(); }
+  inline icache_entry_t* access_icache(reg_t addr)
+  {
+    icache_entry_t* entry = &icache[icache_index(addr)];
+    if (likely(entry->tag == addr))
+      return entry;
+    return refill_icache(addr, entry);
+  }
 
-  // keep the MMU in sync with processor mode
-  void set_supervisor(bool sup) { supervisor = sup; }
-  void set_vm_enabled(bool en) { vm_enabled = en; }
+  inline insn_fetch_t load_insn(reg_t addr)
+  {
+    icache_entry_t entry;
+    return refill_icache(addr, &entry)->data;
+  }
 
-  // flush the TLB and instruction cache
   void flush_tlb();
   void flush_icache();
 
+  void register_memtracer(memtracer_t*);
+
+  int is_dirty_enabled()
+  {
+#ifdef RISCV_ENABLE_DIRTY
+    return 1;
+#else
+    return 0;
+#endif
+  }
+
+  int is_misaligned_enabled()
+  {
+#ifdef RISCV_ENABLE_MISALIGNED
+    return 1;
+#else
+    return 0;
+#endif
+  }
+
 private:
-  char* mem;
-  size_t memsz;
-  reg_t badvaddr;
+  simif_t* sim;
+  processor_t* proc;
+  memtracer_list_t tracer;
+  uint16_t fetch_temp;
 
-  reg_t ptbr;
-  bool supervisor;
-  bool vm_enabled;
+  // implement an instruction cache for simulator performance
+  icache_entry_t icache[ICACHE_ENTRIES];
 
   // implement a TLB for simulator performance
   static const reg_t TLB_ENTRIES = 256;
-  long tlb_data[TLB_ENTRIES];
+  // If a TLB tag has TLB_CHECK_TRIGGERS set, then the MMU must check for a
+  // trigger match before completing an access.
+  static const reg_t TLB_CHECK_TRIGGERS = reg_t(1) << 63;
+  tlb_entry_t tlb_data[TLB_ENTRIES];
   reg_t tlb_insn_tag[TLB_ENTRIES];
   reg_t tlb_load_tag[TLB_ENTRIES];
   reg_t tlb_store_tag[TLB_ENTRIES];
 
-  // implement an instruction cache for simulator performance
-  static const reg_t ICACHE_ENTRIES = 256;
-  insn_t icache_data[ICACHE_ENTRIES];
-  insn_func_t icache_func[ICACHE_ENTRIES];
-  reg_t icache_tag[ICACHE_ENTRIES];
+  // finish translation on a TLB miss and update the TLB
+  tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type);
+  const char* fill_from_mmio(reg_t vaddr, reg_t paddr);
 
-  // finish translation on a TLB miss and upate the TLB
-  void* refill(reg_t addr, bool store, bool fetch);
+  // perform a page table walk for a given VA; set referenced/dirty bits
+  reg_t walk(reg_t addr, access_type type, reg_t prv);
 
-  // perform a page table walk for a given virtual address
-  pte_t walk(reg_t addr);
+  // handle uncommon cases: TLB misses, page faults, MMIO
+  tlb_entry_t fetch_slow_path(reg_t addr);
+  void load_slow_path(reg_t addr, reg_t len, uint8_t* bytes);
+  void store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes);
+  reg_t translate(reg_t addr, access_type type);
 
-  // translate a virtual address to a physical address
-  void* translate(reg_t addr, bool store, bool fetch)
-  {
-    reg_t idx = (addr >> PGSHIFT) % TLB_ENTRIES;
+  // ITLB lookup
+  inline tlb_entry_t translate_insn_addr(reg_t addr) {
+    reg_t vpn = addr >> PGSHIFT;
+    if (likely(tlb_insn_tag[vpn % TLB_ENTRIES] == vpn))
+      return tlb_data[vpn % TLB_ENTRIES];
+    if (unlikely(tlb_insn_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) {
+      uint16_t* ptr = (uint16_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr);
+      int match = proc->trigger_match(OPERATION_EXECUTE, addr, *ptr);
+      if (match >= 0)
+        throw trigger_matched_t(match, OPERATION_EXECUTE, addr, *ptr);
+      return tlb_data[vpn % TLB_ENTRIES];
+    }
+    return fetch_slow_path(addr);
+  }
 
-    reg_t* tlb_tag = fetch ? tlb_insn_tag : store ? tlb_store_tag :tlb_load_tag;
-    reg_t expected_tag = addr & ~(PGSIZE-1);
-    if(likely(tlb_tag[idx] == expected_tag))
-      return (void*)(((long)addr & (PGSIZE-1)) | tlb_data[idx]);
+  inline const uint16_t* translate_insn_addr_to_host(reg_t addr) {
+    return (uint16_t*)(translate_insn_addr(addr).host_offset + addr);
+  }
 
-    return refill(addr, store, fetch);
+  inline trigger_matched_t *trigger_exception(trigger_operation_t operation,
+      reg_t address, reg_t data)
+  {
+    if (!proc) {
+      return NULL;
+    }
+    int match = proc->trigger_match(operation, address, data);
+    if (match == -1)
+      return NULL;
+    if (proc->state.mcontrol[match].timing == 0) {
+      throw trigger_matched_t(match, operation, address, data);
+    }
+    return new trigger_matched_t(match, operation, address, data);
   }
-  
+
+  bool check_triggers_fetch;
+  bool check_triggers_load;
+  bool check_triggers_store;
+  // The exception describing a matched trigger, or NULL.
+  trigger_matched_t *matched_trigger;
+
   friend class processor_t;
 };
 
+struct vm_info {
+  int levels;
+  int idxbits;
+  int ptesize;
+  reg_t ptbase;
+};
+
+inline vm_info decode_vm_info(int xlen, reg_t prv, reg_t satp)
+{
+  if (prv == PRV_M) {
+    return {0, 0, 0, 0};
+  } else if (prv <= PRV_S && xlen == 32) {
+    switch (get_field(satp, SATP32_MODE)) {
+      case SATP_MODE_OFF: return {0, 0, 0, 0};
+      case SATP_MODE_SV32: return {2, 10, 4, (satp & SATP32_PPN) << PGSHIFT};
+      default: abort();
+    }
+  } else if (prv <= PRV_S && xlen == 64) {
+    switch (get_field(satp, SATP64_MODE)) {
+      case SATP_MODE_OFF: return {0, 0, 0, 0};
+      case SATP_MODE_SV39: return {3, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+      case SATP_MODE_SV48: return {4, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+      case SATP_MODE_SV57: return {5, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+      case SATP_MODE_SV64: return {6, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
+      default: abort();
+    }
+  } else {
+    abort();
+  }
+}
+
 #endif