riscv/mmu.h

   1 // See LICENSE for license details.
   2
   3 #ifndef _RISCV_MMU_H
   4 #define _RISCV_MMU_H
   5
   6 #include "decode.h"
   7 #include "trap.h"
   8 #include "common.h"
   9 #include "config.h"
  10 #include "sim.h"
  11 #include "processor.h"
  12 #include "memtracer.h"
  13 #include <stdlib.h>
  14 #include <vector>
  15
  16 // virtual memory configuration
  17 #define PGSHIFT 12
  18 const reg_t PGSIZE = 1 << PGSHIFT;
  19 const reg_t PGMASK = ~(PGSIZE-1);
  20
  21 struct insn_fetch_t
  22 {
  23   insn_func_t func;
  24   insn_t insn;
  25 };
  26
  27 struct icache_entry_t {
  28   reg_t tag;
  29   reg_t pad;
  30   insn_fetch_t data;
  31 };
  32
  33 struct tlb_entry_t {
  34   char* host_offset;
  35   reg_t target_offset;
  36 };
  37
  38 class trigger_matched_t
  39 {
  40   public:
  41     trigger_matched_t(int index,
  42         trigger_operation_t operation, reg_t address, reg_t data) :
  43       index(index), operation(operation), address(address), data(data) {}
  44
  45     int index;
  46     trigger_operation_t operation;
  47     reg_t address;
  48     reg_t data;
  49 };
  50
  51 // this class implements a processor's port into the virtual memory system.
  52 // an MMU and instruction cache are maintained for simulator performance.
  53 class mmu_t
  54 {
  55 public:
  56   mmu_t(sim_t* sim, processor_t* proc);
  57   ~mmu_t();
  58
  59   inline reg_t misaligned_load(reg_t addr, size_t size)
  60   {
  61 #ifdef RISCV_ENABLE_MISALIGNED
  62     reg_t res = 0;
  63     for (size_t i = 0; i < size; i++)
  64       res += (reg_t)load_uint8(addr + i) << (i * 8);
  65     return res;
  66 #else
  67     throw trap_load_address_misaligned(addr);
  68 #endif
  69   }
  70
  71   inline void misaligned_store(reg_t addr, reg_t data, size_t size)
  72   {
  73 #ifdef RISCV_ENABLE_MISALIGNED
  74     for (size_t i = 0; i < size; i++)
  75       store_uint8(addr + i, data >> (i * 8));
  76 #else
  77     throw trap_store_address_misaligned(addr);
  78 #endif
  79   }
  80
  81   // template for functions that load an aligned value from memory
  82   #define load_func(type) \
  83     inline type##_t load_##type(reg_t addr) { \
  84       if (unlikely(addr & (sizeof(type##_t)-1))) \
  85         return misaligned_load(addr, sizeof(type##_t)); \
  86       reg_t vpn = addr >> PGSHIFT; \
  87       if (likely(tlb_load_tag[vpn % TLB_ENTRIES] == vpn)) \
  88         return *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
  89       if (unlikely(tlb_load_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
  90         type##_t data = *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr); \
  91         if (!matched_trigger) { \
  92           matched_trigger = trigger_exception(OPERATION_LOAD, addr, data); \
  93           if (matched_trigger) \
  94             throw *matched_trigger; \
  95         } \
  96         return data; \
  97       } \
  98       type##_t res; \
  99       load_slow_path(addr, sizeof(type##_t), (uint8_t*)&res); \
 100       return res; \
 101     }
 102
 103   // load value from memory at aligned address; zero extend to register width
 104   load_func(uint8)
 105   load_func(uint16)
 106   load_func(uint32)
 107   load_func(uint64)
 108
 109   // load value from memory at aligned address; sign extend to register width
 110   load_func(int8)
 111   load_func(int16)
 112   load_func(int32)
 113   load_func(int64)
 114
 115   // template for functions that store an aligned value to memory
 116   #define store_func(type) \
 117     void store_##type(reg_t addr, type##_t val) { \
 118       if (unlikely(addr & (sizeof(type##_t)-1))) \
 119         return misaligned_store(addr, val, sizeof(type##_t)); \
 120       reg_t vpn = addr >> PGSHIFT; \
 121       if (likely(tlb_store_tag[vpn % TLB_ENTRIES] == vpn)) \
 122         *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
 123       else if (unlikely(tlb_store_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) { \
 124         if (!matched_trigger) { \
 125           matched_trigger = trigger_exception(OPERATION_STORE, addr, val); \
 126           if (matched_trigger) \
 127             throw *matched_trigger; \
 128         } \
 129         *(type##_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr) = val; \
 130       } \
 131       else \
 132         store_slow_path(addr, sizeof(type##_t), (const uint8_t*)&val); \
 133     }
 134
 135   // template for functions that perform an atomic memory operation
 136   #define amo_func(type) \
 137     template<typename op> \
 138     type##_t amo_##type(reg_t addr, op f) { \
 139       if (addr & (sizeof(type##_t)-1)) \
 140         throw trap_store_address_misaligned(addr); \
 141       try { \
 142         auto lhs = load_##type(addr); \
 143         store_##type(addr, f(lhs)); \
 144         return lhs; \
 145       } catch (trap_load_page_fault& t) { \
 146         /* AMO faults should be reported as store faults */ \
 147         throw trap_store_page_fault(t.get_tval()); \
 148       } catch (trap_load_access_fault& t) { \
 149         /* AMO faults should be reported as store faults */ \
 150         throw trap_store_access_fault(t.get_tval()); \
 151       } \
 152     }
 153
 154   void store_float128(reg_t addr, float128_t val)
 155   {
 156 #ifndef RISCV_ENABLE_MISALIGNED
 157     if (unlikely(addr & (sizeof(float128_t)-1)))
 158       throw trap_store_address_misaligned(addr);
 159 #endif
 160     store_uint64(addr, val.v[0]);
 161     store_uint64(addr + 8, val.v[1]);
 162   }
 163
 164   float128_t load_float128(reg_t addr)
 165   {
 166 #ifndef RISCV_ENABLE_MISALIGNED
 167     if (unlikely(addr & (sizeof(float128_t)-1)))
 168       throw trap_load_address_misaligned(addr);
 169 #endif
 170     return (float128_t){load_uint64(addr), load_uint64(addr + 8)};
 171   }
 172
 173   // store value to memory at aligned address
 174   store_func(uint8)
 175   store_func(uint16)
 176   store_func(uint32)
 177   store_func(uint64)
 178
 179   // perform an atomic memory operation at an aligned address
 180   amo_func(uint32)
 181   amo_func(uint64)
 182
 183   static const reg_t ICACHE_ENTRIES = 1024;
 184
 185   inline size_t icache_index(reg_t addr)
 186   {
 187     return (addr / PC_ALIGN) % ICACHE_ENTRIES;
 188   }
 189
 190   inline icache_entry_t* refill_icache(reg_t addr, icache_entry_t* entry)
 191   {
 192     auto tlb_entry = translate_insn_addr(addr);
 193     insn_bits_t insn = *(uint16_t*)(tlb_entry.host_offset + addr);
 194     int length = insn_length(insn);
 195
 196     if (likely(length == 4)) {
 197       insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 2) << 16;
 198     } else if (length == 2) {
 199       insn = (int16_t)insn;
 200     } else if (length == 6) {
 201       insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 4) << 32;
 202       insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16;
 203     } else {
 204       static_assert(sizeof(insn_bits_t) == 8, "insn_bits_t must be uint64_t");
 205       insn |= (insn_bits_t)*(const int16_t*)translate_insn_addr_to_host(addr + 6) << 48;
 206       insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 4) << 32;
 207       insn |= (insn_bits_t)*(const uint16_t*)translate_insn_addr_to_host(addr + 2) << 16;
 208     }
 209
 210     insn_fetch_t fetch = {proc->decode_insn(insn), insn};
 211     entry->tag = addr;
 212     entry->data = fetch;
 213
 214     reg_t paddr = tlb_entry.target_offset + addr;;
 215     if (tracer.interested_in_range(paddr, paddr + 1, FETCH)) {
 216       entry->tag = -1;
 217       tracer.trace(paddr, length, FETCH);
 218     }
 219     return entry;
 220   }
 221
 222   inline icache_entry_t* access_icache(reg_t addr)
 223   {
 224     icache_entry_t* entry = &icache[icache_index(addr)];
 225     if (likely(entry->tag == addr))
 226       return entry;
 227     return refill_icache(addr, entry);
 228   }
 229
 230   inline insn_fetch_t load_insn(reg_t addr)
 231   {
 232     icache_entry_t entry;
 233     return refill_icache(addr, &entry)->data;
 234   }
 235
 236   void flush_tlb();
 237   void flush_icache();
 238
 239   void register_memtracer(memtracer_t*);
 240
 241 private:
 242   sim_t* sim;
 243   processor_t* proc;
 244   memtracer_list_t tracer;
 245   uint16_t fetch_temp;
 246
 247   // implement an instruction cache for simulator performance
 248   icache_entry_t icache[ICACHE_ENTRIES];
 249
 250   // implement a TLB for simulator performance
 251   static const reg_t TLB_ENTRIES = 256;
 252   // If a TLB tag has TLB_CHECK_TRIGGERS set, then the MMU must check for a
 253   // trigger match before completing an access.
 254   static const reg_t TLB_CHECK_TRIGGERS = reg_t(1) << 63;
 255   tlb_entry_t tlb_data[TLB_ENTRIES];
 256   reg_t tlb_insn_tag[TLB_ENTRIES];
 257   reg_t tlb_load_tag[TLB_ENTRIES];
 258   reg_t tlb_store_tag[TLB_ENTRIES];
 259
 260   // finish translation on a TLB miss and update the TLB
 261   tlb_entry_t refill_tlb(reg_t vaddr, reg_t paddr, char* host_addr, access_type type);
 262   const char* fill_from_mmio(reg_t vaddr, reg_t paddr);
 263
 264   // perform a page table walk for a given VA; set referenced/dirty bits
 265   reg_t walk(reg_t addr, access_type type, reg_t prv);
 266
 267   // handle uncommon cases: TLB misses, page faults, MMIO
 268   tlb_entry_t fetch_slow_path(reg_t addr);
 269   void load_slow_path(reg_t addr, reg_t len, uint8_t* bytes);
 270   void store_slow_path(reg_t addr, reg_t len, const uint8_t* bytes);
 271   reg_t translate(reg_t addr, access_type type);
 272
 273   // ITLB lookup
 274   inline tlb_entry_t translate_insn_addr(reg_t addr) {
 275     reg_t vpn = addr >> PGSHIFT;
 276     if (likely(tlb_insn_tag[vpn % TLB_ENTRIES] == vpn))
 277       return tlb_data[vpn % TLB_ENTRIES];
 278     if (unlikely(tlb_insn_tag[vpn % TLB_ENTRIES] == (vpn | TLB_CHECK_TRIGGERS))) {
 279       uint16_t* ptr = (uint16_t*)(tlb_data[vpn % TLB_ENTRIES].host_offset + addr);
 280       int match = proc->trigger_match(OPERATION_EXECUTE, addr, *ptr);
 281       if (match >= 0)
 282         throw trigger_matched_t(match, OPERATION_EXECUTE, addr, *ptr);
 283       return tlb_data[vpn % TLB_ENTRIES];
 284     }
 285     return fetch_slow_path(addr);
 286   }
 287
 288   inline const uint16_t* translate_insn_addr_to_host(reg_t addr) {
 289     return (uint16_t*)(translate_insn_addr(addr).host_offset + addr);
 290   }
 291
 292   inline trigger_matched_t *trigger_exception(trigger_operation_t operation,
 293       reg_t address, reg_t data)
 294   {
 295     if (!proc) {
 296       return NULL;
 297     }
 298     int match = proc->trigger_match(operation, address, data);
 299     if (match == -1)
 300       return NULL;
 301     if (proc->state.mcontrol[match].timing == 0) {
 302       throw trigger_matched_t(match, operation, address, data);
 303     }
 304     return new trigger_matched_t(match, operation, address, data);
 305   }
 306
 307   bool check_triggers_fetch;
 308   bool check_triggers_load;
 309   bool check_triggers_store;
 310   // The exception describing a matched trigger, or NULL.
 311   trigger_matched_t *matched_trigger;
 312
 313   friend class processor_t;
 314 };
 315
 316 struct vm_info {
 317   int levels;
 318   int idxbits;
 319   int ptesize;
 320   reg_t ptbase;
 321 };
 322
 323 inline vm_info decode_vm_info(int xlen, reg_t prv, reg_t satp)
 324 {
 325   if (prv == PRV_M) {
 326     return {0, 0, 0, 0};
 327   } else if (prv <= PRV_S && xlen == 32) {
 328     switch (get_field(satp, SATP32_MODE)) {
 329       case SATP_MODE_OFF: return {0, 0, 0, 0};
 330       case SATP_MODE_SV32: return {2, 10, 4, (satp & SATP32_PPN) << PGSHIFT};
 331       default: abort();
 332     }
 333   } else if (prv <= PRV_S && xlen == 64) {
 334     switch (get_field(satp, SATP64_MODE)) {
 335       case SATP_MODE_OFF: return {0, 0, 0, 0};
 336       case SATP_MODE_SV39: return {3, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
 337       case SATP_MODE_SV48: return {4, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
 338       case SATP_MODE_SV57: return {5, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
 339       case SATP_MODE_SV64: return {6, 9, 8, (satp & SATP64_PPN) << PGSHIFT};
 340       default: abort();
 341     }
 342   } else {
 343     abort();
 344   }
 345 }
 346
 347 #endif