src/soc/minerva/core.py

   1 from functools import reduce
   2 from operator import or_
   3 from itertools import tee
   4
   5 from nmigen import Elaboratable, Module, Record, Mux, Const, Signal, Memory
   6 from nmigen.lib.coding import PriorityEncoder
   7
   8 from soc.minerva.stage import Stage
   9 from soc.minerva.csr import CSRFile
  10 from soc.minerva.units.adder import Adder
  11 from soc.minerva.units.compare import CompareUnit
  12 from soc.minerva.units.debug import DebugUnit
  13 from soc.minerva.units.decoder import InstructionDecoder
  14 from soc.minerva.units.divider import Divider, DummyDivider
  15 from soc.minerva.units.exception import ExceptionUnit
  16 from soc.minerva.units.fetch import BareFetchUnit, CachedFetchUnit, PCSelector
  17 from soc.minerva.units.rvficon import RVFIController, rvfi_layout
  18 from soc.minerva.units.loadstore import (BareLoadStoreUnit, CachedLoadStoreUnit,
  19                                          DataSelector)
  20 from soc.minerva.units.logic import LogicUnit
  21 from soc.minerva.units.multiplier import DummyMultiplier, Multiplier
  22 from soc.minerva.units.predict import BranchPredictor
  23 from soc.minerva.units.shifter import Shifter
  24 from soc.minerva.units.trigger import TriggerUnit
  25
  26 from soc.minerva.units.debug.jtag import jtag_layout
  27 from soc.minerva.wishbone import wishbone_layout
  28
  29
  30 __all__ = ["Minerva"]
  31
  32
  33 _af_layout = [
  34     ("pc",      (33, True)),
  35 ]
  36
  37
  38 _fd_layout = [
  39     ("pc",               32),
  40     ("instruction",      32),
  41     ("fetch_error",       1),
  42     ("fetch_badaddr",    30)
  43 ]
  44
  45
  46 _dx_layout = [
  47     ("pc",                  32),
  48     ("instruction",         32),
  49     ("fetch_error",          1),
  50     ("fetch_badaddr",       30),
  51     ("illegal",              1),
  52     ("rd",                   5),
  53     ("rs1",                  5),
  54     ("rd_we",                1),
  55     ("rs1_re",               1),
  56     ("src1",                32),
  57     ("src2",                32),
  58     ("immediate",           32),
  59     ("bypass_x",             1),
  60     ("bypass_m",             1),
  61     ("funct3",               3),
  62     ("load",                 1),
  63     ("store",                1),
  64     ("adder",                1),
  65     ("adder_sub",            1),
  66     ("logic",                1),
  67     ("multiply",             1),
  68     ("divide",               1),
  69     ("shift",                1),
  70     ("direction",            1),
  71     ("sext",                 1),
  72     ("jump",                 1),
  73     ("compare",              1),
  74     ("branch",               1),
  75     ("branch_target",       32),
  76     ("branch_predict_taken", 1),
  77     ("fence_i",              1),
  78     ("csr",                  1),
  79     ("csr_adr",             12),
  80     ("csr_we",               1),
  81     ("ecall",                1),
  82     ("ebreak",               1),
  83     ("mret",                 1),
  84 ]
  85
  86
  87 _xm_layout = [
  88     ("pc",                  32),
  89     ("instruction",         32),
  90     ("fetch_error",          1),
  91     ("fetch_badaddr",       30),
  92     ("illegal",              1),
  93     ("loadstore_misaligned", 1),
  94     ("ecall",                1),
  95     ("ebreak",               1),
  96     ("rd",                   5),
  97     ("rd_we",                1),
  98     ("bypass_m",             1),
  99     ("funct3",               3),
 100     ("result",              32),
 101     ("shift",                1),
 102     ("load",                 1),
 103     ("store",                1),
 104     ("store_data",          32),
 105     ("compare",              1),
 106     ("multiply",             1),
 107     ("divide",               1),
 108     ("condition_met",        1),
 109     ("branch_target",       32),
 110     ("branch_taken",         1),
 111     ("branch_predict_taken", 1),
 112     ("csr",                  1),
 113     ("csr_adr",             12),
 114     ("csr_we",               1),
 115     ("csr_result",          32),
 116     ("mret",                 1),
 117     ("exception",            1)
 118 ]
 119
 120
 121 _mw_layout = [
 122     ("pc",                32),
 123     ("rd",                 5),
 124     ("rd_we",              1),
 125     ("funct3",             3),
 126     ("result",            32),
 127     ("load",               1),
 128     ("load_data",         32),
 129     ("multiply",           1),
 130     ("exception",          1)
 131 ]
 132
 133
 134 class Minerva(Elaboratable):
 135     def __init__(self, reset_address=0x00000000,
 136                  with_icache=False,
 137                  icache_nways=1, icache_nlines=256, icache_nwords=8, icache_base=0, icache_limit=2**31,
 138                  with_dcache=False,
 139                  dcache_nways=1, dcache_nlines=256, dcache_nwords=8, dcache_base=0, dcache_limit=2**31,
 140                  with_muldiv=False,
 141                  with_debug=False,
 142                  with_trigger=False, nb_triggers=8,
 143                  with_rvfi=False):
 144         self.external_interrupt = Signal(32)
 145         self.timer_interrupt = Signal()
 146         self.software_interrupt = Signal()
 147         self.ibus = Record(wishbone_layout)
 148         self.dbus = Record(wishbone_layout)
 149
 150         if with_debug:
 151             self.jtag = Record(jtag_layout)
 152
 153         if with_rvfi:
 154             self.rvfi = Record(rvfi_layout)
 155
 156         self.reset_address = reset_address
 157         self.with_icache = with_icache
 158         self.icache_args = icache_nways, icache_nlines, icache_nwords, icache_base, icache_limit
 159         self.with_dcache = with_dcache
 160         self.dcache_args = dcache_nways, dcache_nlines, dcache_nwords, dcache_base, dcache_limit
 161         self.with_muldiv = with_muldiv
 162         self.with_debug = with_debug
 163         self.with_trigger = with_trigger
 164         self.nb_triggers = nb_triggers
 165         self.with_rvfi = with_rvfi
 166
 167     def elaborate(self, platform):
 168         cpu = Module()
 169
 170         # pipeline stages
 171
 172         a = cpu.submodules.a = Stage(None, _af_layout)
 173         f = cpu.submodules.f = Stage(_af_layout, _fd_layout)
 174         d = cpu.submodules.d = Stage(_fd_layout, _dx_layout)
 175         x = cpu.submodules.x = Stage(_dx_layout, _xm_layout)
 176         m = cpu.submodules.m = Stage(_xm_layout, _mw_layout)
 177         w = cpu.submodules.w = Stage(_mw_layout, None)
 178         stages = a, f, d, x, m, w
 179
 180         sources, sinks = tee(stages)
 181         next(sinks)
 182         for s1, s2 in zip(sources, sinks):
 183             cpu.d.comb += s1.source.connect(s2.sink)
 184
 185         a.source.pc.reset = self.reset_address - 4
 186         cpu.d.comb += a.valid.eq(Const(1))
 187
 188         # units
 189
 190         pc_sel = cpu.submodules.pc_sel = PCSelector()
 191         data_sel = cpu.submodules.data_sel = DataSelector()
 192         adder = cpu.submodules.adder = Adder()
 193         compare = cpu.submodules.compare = CompareUnit()
 194         decoder = cpu.submodules.decoder = InstructionDecoder(self.with_muldiv)
 195         exception = cpu.submodules.exception = ExceptionUnit()
 196         logic = cpu.submodules.logic = LogicUnit()
 197         predict = cpu.submodules.predict = BranchPredictor()
 198         shifter = cpu.submodules.shifter = Shifter()
 199
 200         if self.with_icache:
 201             fetch = cpu.submodules.fetch = CachedFetchUnit(*self.icache_args)
 202         else:
 203             fetch = cpu.submodules.fetch = BareFetchUnit()
 204
 205         if self.with_dcache:
 206             loadstore = cpu.submodules.loadstore = CachedLoadStoreUnit(
 207                 *self.dcache_args)
 208         else:
 209             loadstore = cpu.submodules.loadstore = BareLoadStoreUnit()
 210
 211         if self.with_muldiv:
 212             multiplier = Multiplier() if not self.with_rvfi else DummyMultiplier()
 213             divider = Divider() if not self.with_rvfi else DummyDivider()
 214             cpu.submodules.multiplier = multiplier
 215             cpu.submodules.divider = divider
 216
 217         if self.with_debug:
 218             debug = cpu.submodules.debug = DebugUnit()
 219
 220         if self.with_trigger:
 221             trigger = cpu.submodules.trigger = TriggerUnit(self.nb_triggers)
 222
 223         if self.with_rvfi:
 224             rvficon = cpu.submodules.rvficon = RVFIController()
 225
 226         # register files
 227
 228         gprf = Memory(width=32, depth=32)
 229         gprf_rp1 = gprf.read_port()
 230         gprf_rp2 = gprf.read_port()
 231         gprf_wp = gprf.write_port()
 232         cpu.submodules += gprf_rp1, gprf_rp2, gprf_wp
 233
 234         csrf = cpu.submodules.csrf = CSRFile()
 235         csrf_rp = csrf.read_port()
 236         csrf_wp = csrf.write_port()
 237
 238         csrf.add_csrs(exception.iter_csrs())
 239         if self.with_debug:
 240             csrf.add_csrs(debug.iter_csrs())
 241         if self.with_trigger:
 242             csrf.add_csrs(trigger.iter_csrs())
 243
 244         # pipeline logic
 245
 246         cpu.d.comb += [
 247             pc_sel.f_pc.eq(f.sink.pc),
 248             pc_sel.d_pc.eq(d.sink.pc),
 249             pc_sel.d_branch_predict_taken.eq(
 250                 predict.d_branch_taken & ~predict.d_fetch_misaligned),
 251             pc_sel.d_branch_target.eq(predict.d_branch_target),
 252             pc_sel.d_valid.eq(d.valid),
 253             pc_sel.x_pc.eq(x.sink.pc),
 254             pc_sel.x_fence_i.eq(x.sink.fence_i),
 255             pc_sel.x_valid.eq(x.valid),
 256             pc_sel.m_branch_predict_taken.eq(m.sink.branch_predict_taken),
 257             pc_sel.m_branch_taken.eq(m.sink.branch_taken),
 258             pc_sel.m_branch_target.eq(m.sink.branch_target),
 259             pc_sel.m_exception.eq(exception.m_raise),
 260             pc_sel.m_mret.eq(m.sink.mret),
 261             pc_sel.m_valid.eq(m.valid),
 262             pc_sel.mtvec_r_base.eq(exception.mtvec.r.base),
 263             pc_sel.mepc_r_base.eq(exception.mepc.r.base)
 264         ]
 265
 266         cpu.d.comb += [
 267             fetch.a_pc.eq(pc_sel.a_pc),
 268             fetch.a_stall.eq(a.stall),
 269             fetch.a_valid.eq(a.valid),
 270             fetch.f_stall.eq(f.stall),
 271             fetch.f_valid.eq(f.valid),
 272             fetch.ibus.connect(self.ibus)
 273         ]
 274
 275         m.stall_on(fetch.a_busy & a.valid)
 276         m.stall_on(fetch.f_busy & f.valid)
 277
 278         if self.with_icache:
 279             flush_icache = x.sink.fence_i & x.valid & ~x.stall
 280             if self.with_debug:
 281                 flush_icache |= debug.resumereq
 282
 283             cpu.d.comb += [
 284                 fetch.a_flush.eq(flush_icache),
 285                 fetch.f_pc.eq(f.sink.pc)
 286             ]
 287
 288         cpu.d.comb += [
 289             decoder.instruction.eq(d.sink.instruction)
 290         ]
 291
 292         if self.with_debug:
 293             with cpu.If(debug.halt & debug.halted):
 294                 cpu.d.comb += gprf_rp1.addr.eq(debug.gprf_addr)
 295             with cpu.Elif(~d.stall):
 296                 cpu.d.comb += gprf_rp1.addr.eq(fetch.f_instruction[15:20])
 297             with cpu.Else():
 298                 cpu.d.comb += gprf_rp1.addr.eq(decoder.rs1)
 299
 300             cpu.d.comb += debug.gprf_dat_r.eq(gprf_rp1.data)
 301         else:
 302             with cpu.If(~d.stall):
 303                 cpu.d.comb += gprf_rp1.addr.eq(fetch.f_instruction[15:20])
 304             with cpu.Else():
 305                 cpu.d.comb += gprf_rp1.addr.eq(decoder.rs1)
 306
 307         with cpu.If(~d.stall):
 308             cpu.d.comb += gprf_rp2.addr.eq(fetch.f_instruction[20:25])
 309         with cpu.Else():
 310             cpu.d.comb += gprf_rp2.addr.eq(decoder.rs2)
 311
 312         with cpu.If(~f.stall):
 313             cpu.d.sync += csrf_rp.addr.eq(fetch.f_instruction[20:32])
 314         cpu.d.comb += csrf_rp.en.eq(decoder.csr & d.valid)
 315
 316         # CSR set/clear instructions are translated to logic operations.
 317         x_csr_set_clear = x.sink.funct3[1]
 318         x_csr_clear = x_csr_set_clear & x.sink.funct3[0]
 319         x_csr_fmt_i = x.sink.funct3[2]
 320         x_csr_src1 = Mux(x_csr_fmt_i, x.sink.rs1, x.sink.src1)
 321         x_csr_src1 = Mux(x_csr_clear, ~x_csr_src1, x_csr_src1)
 322         x_csr_logic_op = x.sink.funct3 | 0b100
 323
 324         cpu.d.comb += [
 325             logic.op.eq(Mux(x.sink.csr, x_csr_logic_op, x.sink.funct3)),
 326             logic.src1.eq(Mux(x.sink.csr, x_csr_src1, x.sink.src1)),
 327             logic.src2.eq(x.sink.src2)
 328         ]
 329
 330         cpu.d.comb += [
 331             adder.sub.eq(x.sink.adder & x.sink.adder_sub |
 332                          x.sink.compare | x.sink.branch),
 333             adder.src1.eq(x.sink.src1),
 334             adder.src2.eq(Mux(x.sink.store, x.sink.immediate, x.sink.src2))
 335         ]
 336
 337         if self.with_muldiv:
 338             cpu.d.comb += [
 339                 multiplier.x_op.eq(x.sink.funct3),
 340                 multiplier.x_src1.eq(x.sink.src1),
 341                 multiplier.x_src2.eq(x.sink.src2),
 342                 multiplier.x_stall.eq(x.stall),
 343                 multiplier.m_stall.eq(m.stall)
 344             ]
 345
 346             cpu.d.comb += [
 347                 divider.x_op.eq(x.sink.funct3),
 348                 divider.x_src1.eq(x.sink.src1),
 349                 divider.x_src2.eq(x.sink.src2),
 350                 divider.x_valid.eq(x.sink.valid),
 351                 divider.x_stall.eq(x.stall)
 352             ]
 353
 354             m.stall_on(divider.m_busy)
 355
 356         cpu.d.comb += [
 357             shifter.x_direction.eq(x.sink.direction),
 358             shifter.x_sext.eq(x.sink.sext),
 359             shifter.x_shamt.eq(x.sink.src2),
 360             shifter.x_src1.eq(x.sink.src1),
 361             shifter.x_stall.eq(x.stall)
 362         ]
 363
 364         cpu.d.comb += [
 365             # compare.op is shared by compare and branch instructions.
 366             compare.op.eq(
 367                 Mux(x.sink.compare, x.sink.funct3 << 1, x.sink.funct3)),
 368             compare.zero.eq(x.sink.src1 == x.sink.src2),
 369             compare.negative.eq(adder.result[-1]),
 370             compare.overflow.eq(adder.overflow),
 371             compare.carry.eq(adder.carry)
 372         ]
 373
 374         cpu.d.comb += [
 375             exception.external_interrupt.eq(self.external_interrupt),
 376             exception.timer_interrupt.eq(self.timer_interrupt),
 377             exception.software_interrupt.eq(self.software_interrupt),
 378             exception.m_fetch_misaligned.eq(
 379                 m.sink.branch_taken & m.sink.branch_target[:2].bool()),
 380             exception.m_fetch_error.eq(m.sink.fetch_error),
 381             exception.m_fetch_badaddr.eq(m.sink.fetch_badaddr),
 382             exception.m_load_misaligned.eq(
 383                 m.sink.load & m.sink.loadstore_misaligned),
 384             exception.m_load_error.eq(loadstore.m_load_error),
 385             exception.m_store_misaligned.eq(
 386                 m.sink.store & m.sink.loadstore_misaligned),
 387             exception.m_store_error.eq(loadstore.m_store_error),
 388             exception.m_loadstore_badaddr.eq(loadstore.m_badaddr),
 389             exception.m_branch_target.eq(m.sink.branch_target),
 390             exception.m_illegal.eq(m.sink.illegal),
 391             exception.m_ecall.eq(m.sink.ecall),
 392             exception.m_pc.eq(m.sink.pc),
 393             exception.m_instruction.eq(m.sink.instruction),
 394             exception.m_result.eq(m.sink.result),
 395             exception.m_mret.eq(m.sink.mret),
 396             exception.m_stall.eq(m.sink.stall),
 397             exception.m_valid.eq(m.valid)
 398         ]
 399
 400         m_ebreak = m.sink.ebreak
 401         if self.with_debug:
 402             # If dcsr.ebreakm is set, EBREAK instructions enter Debug Mode.
 403             # We do not want to raise an exception in this case because Debug Mode
 404             # should be invisible to software execution.
 405             m_ebreak &= ~debug.dcsr_ebreakm
 406         if self.with_trigger:
 407             m_ebreak |= trigger.trap
 408         cpu.d.comb += exception.m_ebreak.eq(m_ebreak)
 409
 410         m.kill_on(m.source.exception & m.source.valid)
 411
 412         cpu.d.comb += [
 413             data_sel.x_offset.eq(adder.result[:2]),
 414             data_sel.x_funct3.eq(x.sink.funct3),
 415             data_sel.x_store_operand.eq(x.sink.src2),
 416             data_sel.w_offset.eq(w.sink.result[:2]),
 417             data_sel.w_funct3.eq(w.sink.funct3),
 418             data_sel.w_load_data.eq(w.sink.load_data)
 419         ]
 420
 421         cpu.d.comb += [
 422             loadstore.x_addr.eq(adder.result),
 423             loadstore.x_mask.eq(data_sel.x_mask),
 424             loadstore.x_load.eq(x.sink.load),
 425             loadstore.x_store.eq(x.sink.store),
 426             loadstore.x_store_data.eq(data_sel.x_store_data),
 427             loadstore.x_stall.eq(x.stall),
 428             loadstore.x_valid.eq(x.valid),
 429             loadstore.m_stall.eq(m.stall),
 430             loadstore.m_valid.eq(m.valid)
 431         ]
 432
 433         m.stall_on(loadstore.x_busy & x.valid)
 434         m.stall_on(loadstore.m_busy & m.valid)
 435
 436         if self.with_dcache:
 437             if self.with_debug:
 438                 cpu.d.comb += loadstore.x_flush.eq(debug.resumereq)
 439
 440             cpu.d.comb += [
 441                 loadstore.x_fence_i.eq(x.sink.fence_i),
 442                 loadstore.m_addr.eq(m.sink.result),
 443                 loadstore.m_load.eq(m.sink.load),
 444                 loadstore.m_store.eq(m.sink.store),
 445             ]
 446
 447             x.stall_on(loadstore.x_busy & x.valid)
 448
 449         for s in a, f:
 450             s.kill_on(x.sink.fence_i & x.valid)
 451
 452         if self.with_debug:
 453             with cpu.If(debug.halt & debug.halted):
 454                 cpu.d.comb += debug.dbus.connect(self.dbus)
 455             with cpu.Else():
 456                 cpu.d.comb += loadstore.dbus.connect(self.dbus)
 457         else:
 458             cpu.d.comb += loadstore.dbus.connect(self.dbus)
 459
 460         # RAW hazard management
 461
 462         x_raw_rs1 = Signal()
 463         m_raw_rs1 = Signal()
 464         w_raw_rs1 = Signal()
 465         x_raw_rs2 = Signal()
 466         m_raw_rs2 = Signal()
 467         w_raw_rs2 = Signal()
 468
 469         x_raw_csr = Signal()
 470         m_raw_csr = Signal()
 471
 472         x_lock = Signal()
 473         m_lock = Signal()
 474
 475         cpu.d.comb += [
 476             x_raw_rs1.eq((x.sink.rd != 0) & (
 477                 x.sink.rd == decoder.rs1) & x.sink.rd_we),
 478             m_raw_rs1.eq((m.sink.rd != 0) & (
 479                 m.sink.rd == decoder.rs1) & m.sink.rd_we),
 480             w_raw_rs1.eq((w.sink.rd != 0) & (
 481                 w.sink.rd == decoder.rs1) & w.sink.rd_we),
 482
 483             x_raw_rs2.eq((x.sink.rd != 0) & (
 484                 x.sink.rd == decoder.rs2) & x.sink.rd_we),
 485             m_raw_rs2.eq((m.sink.rd != 0) & (
 486                 m.sink.rd == decoder.rs2) & m.sink.rd_we),
 487             w_raw_rs2.eq((w.sink.rd != 0) & (
 488                 w.sink.rd == decoder.rs2) & w.sink.rd_we),
 489
 490             x_raw_csr.eq((x.sink.csr_adr == decoder.immediate)
 491                          & x.sink.csr_we),
 492             m_raw_csr.eq((m.sink.csr_adr == decoder.immediate)
 493                          & m.sink.csr_we),
 494
 495             x_lock.eq(~x.sink.bypass_x & (decoder.rs1_re & x_raw_rs1 |
 496                                           decoder.rs2_re & x_raw_rs2) | decoder.csr & x_raw_csr),
 497             m_lock.eq(~m.sink.bypass_m & (decoder.rs1_re & m_raw_rs1 |
 498                                           decoder.rs2_re & m_raw_rs2) | decoder.csr & m_raw_csr)
 499         ]
 500
 501         if self.with_debug:
 502             d.stall_on((x_lock & x.valid | m_lock & m.valid)
 503                        & d.valid & ~debug.dcsr_step)
 504         else:
 505             d.stall_on((x_lock & x.valid | m_lock & m.valid) & d.valid)
 506
 507         # result selection
 508
 509         x_result = Signal(32)
 510         m_result = Signal(32)
 511         w_result = Signal(32)
 512         x_csr_result = Signal(32)
 513
 514         with cpu.If(x.sink.jump):
 515             cpu.d.comb += x_result.eq(x.sink.pc + 4)
 516         with cpu.Elif(x.sink.logic):
 517             cpu.d.comb += x_result.eq(logic.result)
 518         with cpu.Elif(x.sink.csr):
 519             cpu.d.comb += x_result.eq(x.sink.src2)
 520         with cpu.Else():
 521             cpu.d.comb += x_result.eq(adder.result)
 522
 523         with cpu.If(m.sink.compare):
 524             cpu.d.comb += m_result.eq(m.sink.condition_met)
 525         if self.with_muldiv:
 526             with cpu.Elif(m.sink.divide):
 527                 cpu.d.comb += m_result.eq(divider.m_result)
 528         with cpu.Elif(m.sink.shift):
 529             cpu.d.comb += m_result.eq(shifter.m_result)
 530         with cpu.Else():
 531             cpu.d.comb += m_result.eq(m.sink.result)
 532
 533         with cpu.If(w.sink.load):
 534             cpu.d.comb += w_result.eq(data_sel.w_load_result)
 535         if self.with_muldiv:
 536             with cpu.Elif(w.sink.multiply):
 537                 cpu.d.comb += w_result.eq(multiplier.w_result)
 538         with cpu.Else():
 539             cpu.d.comb += w_result.eq(w.sink.result)
 540
 541         with cpu.If(x_csr_set_clear):
 542             cpu.d.comb += x_csr_result.eq(logic.result)
 543         with cpu.Else():
 544             cpu.d.comb += x_csr_result.eq(x_csr_src1)
 545
 546         cpu.d.comb += [
 547             csrf_wp.en.eq(m.sink.csr & m.sink.csr_we & m.valid &
 548                           ~exception.m_raise & ~m.stall),
 549             csrf_wp.addr.eq(m.sink.csr_adr),
 550             csrf_wp.data.eq(m.sink.csr_result)
 551         ]
 552
 553         if self.with_debug:
 554             with cpu.If(debug.halt & debug.halted):
 555                 cpu.d.comb += [
 556                     gprf_wp.addr.eq(debug.gprf_addr),
 557                     gprf_wp.en.eq(debug.gprf_we),
 558                     gprf_wp.data.eq(debug.gprf_dat_w)
 559                 ]
 560             with cpu.Else():
 561                 cpu.d.comb += [
 562                     gprf_wp.en.eq((w.sink.rd != 0) & w.sink.rd_we &
 563                                   w.valid & ~w.sink.exception),
 564                     gprf_wp.addr.eq(w.sink.rd),
 565                     gprf_wp.data.eq(w_result)
 566                 ]
 567         else:
 568             cpu.d.comb += [
 569                 gprf_wp.en.eq((w.sink.rd != 0) & w.sink.rd_we & w.valid),
 570                 gprf_wp.addr.eq(w.sink.rd),
 571                 gprf_wp.data.eq(w_result)
 572             ]
 573
 574         # D stage operand selection
 575
 576         d_src1 = Signal(32)
 577         d_src2 = Signal(32)
 578
 579         with cpu.If(decoder.lui):
 580             cpu.d.comb += d_src1.eq(0)
 581         with cpu.Elif(decoder.auipc):
 582             cpu.d.comb += d_src1.eq(d.sink.pc)
 583         with cpu.Elif(decoder.rs1_re & (decoder.rs1 == 0)):
 584             cpu.d.comb += d_src1.eq(0)
 585         with cpu.Elif(x_raw_rs1 & x.valid):
 586             cpu.d.comb += d_src1.eq(x_result)
 587         with cpu.Elif(m_raw_rs1 & m.valid):
 588             cpu.d.comb += d_src1.eq(m_result)
 589         with cpu.Elif(w_raw_rs1 & w.valid):
 590             cpu.d.comb += d_src1.eq(w_result)
 591         with cpu.Else():
 592             cpu.d.comb += d_src1.eq(gprf_rp1.data)
 593
 594         with cpu.If(decoder.csr):
 595             cpu.d.comb += d_src2.eq(csrf_rp.data)
 596         with cpu.Elif(~decoder.rs2_re):
 597             cpu.d.comb += d_src2.eq(decoder.immediate)
 598         with cpu.Elif(decoder.rs2 == 0):
 599             cpu.d.comb += d_src2.eq(0)
 600         with cpu.Elif(x_raw_rs2 & x.valid):
 601             cpu.d.comb += d_src2.eq(x_result)
 602         with cpu.Elif(m_raw_rs2 & m.valid):
 603             cpu.d.comb += d_src2.eq(m_result)
 604         with cpu.Elif(w_raw_rs2 & w.valid):
 605             cpu.d.comb += d_src2.eq(w_result)
 606         with cpu.Else():
 607             cpu.d.comb += d_src2.eq(gprf_rp2.data)
 608
 609         # branch prediction
 610
 611         cpu.d.comb += [
 612             predict.d_branch.eq(decoder.branch),
 613             predict.d_jump.eq(decoder.jump),
 614             predict.d_offset.eq(decoder.immediate),
 615             predict.d_pc.eq(d.sink.pc),
 616             predict.d_rs1_re.eq(decoder.rs1_re)
 617         ]
 618
 619         a.kill_on(predict.d_branch_taken & ~
 620                   predict.d_fetch_misaligned & d.valid)
 621         for s in a, f:
 622             s.kill_on(m.sink.branch_predict_taken & ~
 623                       m.sink.branch_taken & m.valid)
 624         for s in a, f, d:
 625             s.kill_on(~m.sink.branch_predict_taken &
 626                       m.sink.branch_taken & m.valid)
 627             s.kill_on((exception.m_raise | m.sink.mret) & m.valid)
 628
 629         # debug unit
 630
 631         if self.with_debug:
 632             cpu.d.comb += [
 633                 debug.jtag.connect(self.jtag),
 634                 debug.x_pc.eq(x.sink.pc),
 635                 debug.x_ebreak.eq(x.sink.ebreak),
 636                 debug.x_stall.eq(x.stall),
 637                 debug.m_branch_taken.eq(m.sink.branch_taken),
 638                 debug.m_branch_target.eq(m.sink.branch_target),
 639                 debug.m_mret.eq(m.sink.mret),
 640                 debug.m_exception.eq(exception.m_raise),
 641                 debug.m_pc.eq(m.sink.pc),
 642                 debug.m_valid.eq(m.valid),
 643                 debug.mepc_r_base.eq(exception.mepc.r.base),
 644                 debug.mtvec_r_base.eq(exception.mtvec.r.base)
 645             ]
 646
 647             if self.with_trigger:
 648                 cpu.d.comb += debug.trigger_haltreq.eq(trigger.haltreq)
 649             else:
 650                 cpu.d.comb += debug.trigger_haltreq.eq(Const(0))
 651
 652             csrf_debug_rp = csrf.read_port()
 653             csrf_debug_wp = csrf.write_port()
 654             cpu.d.comb += [
 655                 csrf_debug_rp.addr.eq(debug.csrf_addr),
 656                 csrf_debug_rp.en.eq(debug.csrf_re),
 657                 debug.csrf_dat_r.eq(csrf_debug_rp.data),
 658                 csrf_debug_wp.addr.eq(debug.csrf_addr),
 659                 csrf_debug_wp.en.eq(debug.csrf_we),
 660                 csrf_debug_wp.data.eq(debug.csrf_dat_w)
 661             ]
 662
 663             x.stall_on(debug.halt)
 664             m.stall_on(debug.dcsr_step & m.valid & ~debug.halt)
 665             for s in a, f, d, x:
 666                 s.kill_on(debug.killall)
 667
 668             halted = x.stall & ~reduce(or_, (s.valid for s in (m, w)))
 669             cpu.d.sync += debug.halted.eq(halted)
 670
 671             with cpu.If(debug.resumereq):
 672                 with cpu.If(~debug.dbus_busy):
 673                     cpu.d.comb += debug.resumeack.eq(1)
 674                     cpu.d.sync += a.source.pc.eq(debug.dpc_value - 4)
 675
 676         if self.with_trigger:
 677             cpu.d.comb += [
 678                 trigger.x_pc.eq(x.sink.pc),
 679                 trigger.x_valid.eq(x.valid),
 680             ]
 681
 682         if self.with_rvfi:
 683             cpu.d.comb += [
 684                 rvficon.d_insn.eq(decoder.instruction),
 685                 rvficon.d_rs1_addr.eq(Mux(decoder.rs1_re, decoder.rs1, 0)),
 686                 rvficon.d_rs2_addr.eq(Mux(decoder.rs2_re, decoder.rs2, 0)),
 687                 rvficon.d_rs1_rdata.eq(Mux(decoder.rs1_re, d_src1, 0)),
 688                 rvficon.d_rs2_rdata.eq(Mux(decoder.rs2_re, d_src2, 0)),
 689                 rvficon.d_stall.eq(d.stall),
 690                 rvficon.x_mem_addr.eq(loadstore.x_addr[2:] << 2),
 691                 rvficon.x_mem_wmask.eq(
 692                     Mux(loadstore.x_store, loadstore.x_mask, 0)),
 693                 rvficon.x_mem_rmask.eq(
 694                     Mux(loadstore.x_load, loadstore.x_mask, 0)),
 695                 rvficon.x_mem_wdata.eq(loadstore.x_store_data),
 696                 rvficon.x_stall.eq(x.stall),
 697                 rvficon.m_mem_rdata.eq(loadstore.m_load_data),
 698                 rvficon.m_fetch_misaligned.eq(exception.m_fetch_misaligned),
 699                 rvficon.m_illegal_insn.eq(m.sink.illegal),
 700                 rvficon.m_load_misaligned.eq(exception.m_load_misaligned),
 701                 rvficon.m_store_misaligned.eq(exception.m_store_misaligned),
 702                 rvficon.m_exception.eq(exception.m_raise),
 703                 rvficon.m_mret.eq(m.sink.mret),
 704                 rvficon.m_branch_taken.eq(m.sink.branch_taken),
 705                 rvficon.m_branch_target.eq(m.sink.branch_target),
 706                 rvficon.m_pc_rdata.eq(m.sink.pc),
 707                 rvficon.m_stall.eq(m.stall),
 708                 rvficon.m_valid.eq(m.valid),
 709                 rvficon.w_rd_addr.eq(Mux(gprf_wp.en, gprf_wp.addr, 0)),
 710                 rvficon.w_rd_wdata.eq(Mux(gprf_wp.en, gprf_wp.data, 0)),
 711                 rvficon.mtvec_r_base.eq(exception.mtvec.r.base),
 712                 rvficon.mepc_r_value.eq(exception.mepc.r),
 713                 rvficon.rvfi.connect(self.rvfi)
 714             ]
 715
 716         # pipeline registers
 717
 718         # A/F
 719         with cpu.If(~a.stall):
 720             cpu.d.sync += a.source.pc.eq(fetch.a_pc)
 721
 722         # F/D
 723         with cpu.If(~f.stall):
 724             cpu.d.sync += [
 725                 f.source.pc.eq(f.sink.pc),
 726                 f.source.instruction.eq(fetch.f_instruction),
 727                 f.source.fetch_error.eq(fetch.f_fetch_error),
 728                 f.source.fetch_badaddr.eq(fetch.f_badaddr)
 729             ]
 730
 731         # D/X
 732         with cpu.If(~d.stall):
 733             cpu.d.sync += [
 734                 d.source.pc.eq(d.sink.pc),
 735                 d.source.instruction.eq(d.sink.instruction),
 736                 d.source.fetch_error.eq(d.sink.fetch_error),
 737                 d.source.fetch_badaddr.eq(d.sink.fetch_badaddr),
 738                 d.source.illegal.eq(decoder.illegal),
 739                 d.source.rd.eq(decoder.rd),
 740                 d.source.rs1.eq(decoder.rs1),
 741                 d.source.rd_we.eq(decoder.rd_we),
 742                 d.source.rs1_re.eq(decoder.rs1_re),
 743                 d.source.immediate.eq(decoder.immediate),
 744                 d.source.bypass_x.eq(decoder.bypass_x),
 745                 d.source.bypass_m.eq(decoder.bypass_m),
 746                 d.source.funct3.eq(decoder.funct3),
 747                 d.source.load.eq(decoder.load),
 748                 d.source.store.eq(decoder.store),
 749                 d.source.adder.eq(decoder.adder),
 750                 d.source.adder_sub.eq(decoder.adder_sub),
 751                 d.source.compare.eq(decoder.compare),
 752                 d.source.logic.eq(decoder.logic),
 753                 d.source.shift.eq(decoder.shift),
 754                 d.source.direction.eq(decoder.direction),
 755                 d.source.sext.eq(decoder.sext),
 756                 d.source.jump.eq(decoder.jump),
 757                 d.source.branch.eq(decoder.branch),
 758                 d.source.fence_i.eq(decoder.fence_i),
 759                 d.source.csr.eq(decoder.csr),
 760                 d.source.csr_adr.eq(decoder.immediate),
 761                 d.source.csr_we.eq(decoder.csr_we),
 762                 d.source.ecall.eq(decoder.ecall),
 763                 d.source.ebreak.eq(decoder.ebreak),
 764                 d.source.mret.eq(decoder.mret),
 765                 d.source.src1.eq(d_src1),
 766                 d.source.src2.eq(d_src2),
 767                 d.source.branch_predict_taken.eq(
 768                     predict.d_branch_taken & ~predict.d_fetch_misaligned),
 769                 d.source.branch_target.eq(predict.d_branch_target)
 770             ]
 771             if self.with_muldiv:
 772                 cpu.d.sync += [
 773                     d.source.multiply.eq(decoder.multiply),
 774                     d.source.divide.eq(decoder.divide)
 775                 ]
 776
 777         # X/M
 778         with cpu.If(~x.stall):
 779             cpu.d.sync += [
 780                 x.source.pc.eq(x.sink.pc),
 781                 x.source.instruction.eq(x.sink.instruction),
 782                 x.source.fetch_error.eq(x.sink.fetch_error),
 783                 x.source.fetch_badaddr.eq(x.sink.fetch_badaddr),
 784                 x.source.illegal.eq(x.sink.illegal),
 785                 x.source.loadstore_misaligned.eq(data_sel.x_misaligned),
 786                 x.source.ecall.eq(x.sink.ecall),
 787                 x.source.ebreak.eq(x.sink.ebreak),
 788                 x.source.rd.eq(x.sink.rd),
 789                 x.source.rd_we.eq(x.sink.rd_we),
 790                 x.source.bypass_m.eq(x.sink.bypass_m | x.sink.bypass_x),
 791                 x.source.funct3.eq(x.sink.funct3),
 792                 x.source.load.eq(x.sink.load),
 793                 x.source.store.eq(x.sink.store),
 794                 x.source.store_data.eq(loadstore.x_store_data),
 795                 x.source.compare.eq(x.sink.compare),
 796                 x.source.shift.eq(x.sink.shift),
 797                 x.source.mret.eq(x.sink.mret),
 798                 x.source.condition_met.eq(compare.condition_met),
 799                 x.source.branch_taken.eq(
 800                     x.sink.jump | x.sink.branch & compare.condition_met),
 801                 x.source.branch_target.eq(
 802                     Mux(x.sink.jump & x.sink.rs1_re, adder.result[1:] << 1, x.sink.branch_target)),
 803                 x.source.branch_predict_taken.eq(x.sink.branch_predict_taken),
 804                 x.source.csr.eq(x.sink.csr),
 805                 x.source.csr_adr.eq(x.sink.csr_adr),
 806                 x.source.csr_we.eq(x.sink.csr_we),
 807                 x.source.csr_result.eq(x_csr_result),
 808                 x.source.result.eq(x_result)
 809             ]
 810             if self.with_muldiv:
 811                 cpu.d.sync += [
 812                     x.source.multiply.eq(x.sink.multiply),
 813                     x.source.divide.eq(x.sink.divide)
 814                 ]
 815
 816         # M/W
 817         with cpu.If(~m.stall):
 818             cpu.d.sync += [
 819                 m.source.pc.eq(m.sink.pc),
 820                 m.source.rd.eq(m.sink.rd),
 821                 m.source.load.eq(m.sink.load),
 822                 m.source.funct3.eq(m.sink.funct3),
 823                 m.source.load_data.eq(loadstore.m_load_data),
 824                 m.source.rd_we.eq(m.sink.rd_we),
 825                 m.source.result.eq(m_result),
 826                 m.source.exception.eq(exception.m_raise)
 827             ]
 828             if self.with_muldiv:
 829                 cpu.d.sync += [
 830                     m.source.multiply.eq(m.sink.multiply)
 831                 ]
 832
 833         return cpu