src/soc/fu/ldst/loadstore.py

   1 """LoadStore1 FSM.
   2
   3 based on microwatt loadstore1.vhdl, but conforming to PortInterface.
   4 unlike loadstore1.vhdl this does *not* deal with actual Load/Store
   5 ops: that job is handled by LDSTCompUnit, which talks to LoadStore1
   6 by way of PortInterface.  PortInterface is where things need extending,
   7 such as adding dcbz support, etc.
   8
   9 this module basically handles "pure" load / store operations, and
  10 its first job is to ask the D-Cache for the data.  if that fails,
  11 the second task (if virtual memory is enabled) is to ask the MMU
  12 to perform a TLB, then to go *back* to the cache and ask again.
  13
  14 Links:
  15
  16 * https://bugs.libre-soc.org/show_bug.cgi?id=465
  17
  18 """
  19
  20 from nmigen import (Elaboratable, Module, Signal, Shape, unsigned, Cat, Mux,
  21                     Record, Memory,
  22                     Const)
  23 from nmutil.iocontrol import RecordObject
  24 from nmutil.util import rising_edge, Display
  25 from enum import Enum, unique
  26
  27 from soc.experiment.dcache import DCache
  28 from soc.experiment.icache import ICache
  29 from soc.experiment.pimem import PortInterfaceBase
  30 from soc.experiment.mem_types import LoadStore1ToMMUType
  31 from soc.experiment.mem_types import MMUToLoadStore1Type
  32
  33 from soc.minerva.wishbone import make_wb_layout
  34 from soc.bus.sram import SRAM
  35 from nmutil.util import Display
  36
  37
  38 @unique
  39 class State(Enum):
  40     IDLE = 0       # ready for instruction
  41     ACK_WAIT = 1   # waiting for ack from dcache
  42     MMU_LOOKUP = 2 # waiting for MMU to look up translation
  43     TLBIE_WAIT = 3 # waiting for MMU to finish doing a tlbie
  44
  45
  46 # captures the LDSTRequest from the PortInterface, which "blips" most
  47 # of this at us (pipeline-style).
  48 class LDSTRequest(RecordObject):
  49     def __init__(self, name=None):
  50         RecordObject.__init__(self, name=name)
  51
  52         self.load          = Signal()
  53         self.dcbz          = Signal()
  54         self.addr          = Signal(64)
  55         # self.store_data    = Signal(64) # this is already sync (on a delay)
  56         self.byte_sel      = Signal(8)
  57         self.nc            = Signal()              # non-cacheable access
  58         self.virt_mode     = Signal()
  59         self.priv_mode     = Signal()
  60         self.mode_32bit    = Signal() # XXX UNUSED AT PRESENT
  61         self.align_intr    = Signal()
  62
  63
  64 # glue logic for microwatt mmu and dcache
  65 class LoadStore1(PortInterfaceBase):
  66     def __init__(self, pspec):
  67         self.pspec = pspec
  68         self.disable_cache = (hasattr(pspec, "disable_cache") and
  69                               pspec.disable_cache == True)
  70         regwid = pspec.reg_wid
  71         addrwid = pspec.addr_wid
  72
  73         super().__init__(regwid, addrwid)
  74         self.dcache = DCache()
  75         self.icache = ICache(pspec)
  76         # these names are from the perspective of here (LoadStore1)
  77         self.d_out  = self.dcache.d_in     # in to dcache is out for LoadStore
  78         self.d_in = self.dcache.d_out      # out from dcache is in for LoadStore
  79         self.i_out  = self.icache.i_in     # in to icache is out for LoadStore
  80         self.i_in = self.icache.i_out      # out from icache is in for LoadStore
  81         self.m_out  = LoadStore1ToMMUType("m_out") # out *to* MMU
  82         self.m_in = MMUToLoadStore1Type("m_in")   # in *from* MMU
  83         self.req = LDSTRequest(name="ldst_req")
  84
  85         # TODO, convert dcache wb_in/wb_out to "standard" nmigen Wishbone bus
  86         self.dbus = Record(make_wb_layout(pspec))
  87         self.ibus = Record(make_wb_layout(pspec))
  88
  89         # for creating a single clock blip to DCache
  90         self.d_valid = Signal()
  91         self.d_w_valid = Signal()
  92         self.d_validblip = Signal()
  93
  94         # state info for LD/ST
  95         self.done          = Signal()
  96         self.done_delay    = Signal()
  97         # latch most of the input request
  98         self.load          = Signal()
  99         self.tlbie         = Signal()
 100         self.dcbz          = Signal()
 101         self.addr          = Signal(64)
 102         self.maddr          = Signal(64)
 103         self.store_data    = Signal(64)
 104         self.load_data     = Signal(64)
 105         self.load_data_delay = Signal(64)
 106         self.byte_sel      = Signal(8)
 107         #self.xerc         : xer_common_t;
 108         #self.reserve       = Signal()
 109         #self.atomic        = Signal()
 110         #self.atomic_last   = Signal()
 111         #self.rc            = Signal()
 112         self.nc            = Signal()              # non-cacheable access
 113         self.virt_mode     = Signal()
 114         self.priv_mode     = Signal()
 115         self.mode_32bit    = Signal() # XXX UNUSED AT PRESENT
 116         self.state         = Signal(State)
 117         self.instr_fault   = Signal()  # indicator to request i-cache MMU lookup
 118         self.r_instr_fault  = Signal() # accessed in external_busy
 119         self.align_intr    = Signal()
 120         self.busy          = Signal()
 121         self.wait_dcache   = Signal()
 122         self.wait_mmu      = Signal()
 123         #self.intr_vec     : integer range 0 to 16#fff#;
 124         #self.nia           = Signal(64)
 125         #self.srr1          = Signal(16)
 126         # use these to set the dsisr or dar respectively
 127         self.mmu_set_spr    = Signal()
 128         self.mmu_set_dsisr  = Signal()
 129         self.mmu_set_dar    = Signal()
 130         self.sprval_in      = Signal(64)
 131
 132         # ONLY access these read-only, do NOT attempt to change
 133         self.dsisr          = Signal(32)
 134         self.dar            = Signal(64)
 135
 136     # when external_busy set, do not allow PortInterface to proceed
 137     def external_busy(self, m):
 138         return self.instr_fault | self.r_instr_fault
 139
 140     def set_wr_addr(self, m, addr, mask, misalign, msr, is_dcbz):
 141         m.d.comb += self.req.load.eq(0) # store operation
 142         m.d.comb += self.req.byte_sel.eq(mask)
 143         m.d.comb += self.req.addr.eq(addr)
 144         m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem  ==> priv
 145         m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
 146         m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
 147         m.d.comb += self.req.align_intr.eq(misalign)
 148         m.d.comb += self.req.dcbz.eq(is_dcbz)
 149
 150         # m.d.comb += Display("set_wr_addr %i dcbz %i",addr,is_dcbz)
 151
 152         # option to disable the cache entirely for write
 153         if self.disable_cache:
 154             m.d.comb += self.req.nc.eq(1)
 155         return None
 156
 157     def set_rd_addr(self, m, addr, mask, misalign, msr):
 158         m.d.comb += self.d_valid.eq(1)
 159         m.d.comb += self.req.load.eq(1) # load operation
 160         m.d.comb += self.req.byte_sel.eq(mask)
 161         m.d.comb += self.req.align_intr.eq(misalign)
 162         m.d.comb += self.req.addr.eq(addr)
 163         m.d.comb += self.req.priv_mode.eq(~msr.pr) # not-problem  ==> priv
 164         m.d.comb += self.req.virt_mode.eq(msr.dr) # DR ==> virt
 165         m.d.comb += self.req.mode_32bit.eq(~msr.sf) # not-sixty-four ==> 32bit
 166         # BAD HACK! disable cacheing on LD when address is 0xCxxx_xxxx
 167         # this is for peripherals. same thing done in Microwatt loadstore1.vhdl
 168         with m.If(addr[28:] == Const(0xc, 4)):
 169             m.d.comb += self.req.nc.eq(1)
 170         # option to disable the cache entirely for read
 171         if self.disable_cache:
 172             m.d.comb += self.req.nc.eq(1)
 173         return None #FIXME return value
 174
 175     def set_wr_data(self, m, data, wen):
 176         # do the "blip" on write data
 177         m.d.comb += self.d_valid.eq(1)
 178         # put data into comb which is picked up in main elaborate()
 179         m.d.comb += self.d_w_valid.eq(1)
 180         m.d.comb += self.store_data.eq(data)
 181         #m.d.sync += self.d_out.byte_sel.eq(wen) # this might not be needed
 182         st_ok = self.done # TODO indicates write data is valid
 183         return st_ok
 184
 185     def get_rd_data(self, m):
 186         ld_ok = self.done_delay # indicates read data is valid
 187         data = self.load_data_delay   # actual read data
 188         return data, ld_ok
 189
 190     def elaborate(self, platform):
 191         m = super().elaborate(platform)
 192         comb, sync = m.d.comb, m.d.sync
 193
 194         # microwatt takes one more cycle before next operation can be issued
 195         sync += self.done_delay.eq(self.done)
 196         sync += self.load_data_delay.eq(self.load_data)
 197
 198         # create dcache and icache module
 199         m.submodules.dcache = dcache = self.dcache
 200         m.submodules.icache = icache = self.icache
 201
 202         # temp vars
 203         d_out, d_in, dbus = self.d_out, self.d_in, self.dbus
 204         i_out, i_in, ibus = self.i_out, self.i_in, self.ibus
 205         m_out, m_in = self.m_out, self.m_in
 206         exc = self.pi.exc_o
 207         exception = exc.happened
 208         mmureq = Signal()
 209
 210         # copy of address, but gets over-ridden for instr_fault
 211         maddr = Signal(64)
 212         m.d.comb += maddr.eq(self.addr)
 213
 214         # create a blip (single pulse) on valid read/write request
 215         # this can be over-ridden in the FSM to get dcache to re-run
 216         # a request when MMU_LOOKUP completes.
 217         m.d.comb += self.d_validblip.eq(rising_edge(m, self.d_valid))
 218         ldst_r = LDSTRequest("ldst_r")
 219         comb += Display("MMUTEST: LoadStore1 d_in.error=%i",d_in.error)
 220
 221         # fsm skeleton
 222         with m.Switch(self.state):
 223             with m.Case(State.IDLE):
 224                 with m.If((self.d_validblip | self.instr_fault) &
 225                           ~exc.happened):
 226                     comb += self.busy.eq(1)
 227                     sync += self.state.eq(State.ACK_WAIT)
 228                     sync += ldst_r.eq(self.req) # copy of LDSTRequest on "blip"
 229 #                   sync += Display("validblip self.req.virt_mode=%i",
 230 #                   self.req.virt_mode)
 231                     with m.If(self.instr_fault):
 232                         comb += mmureq.eq(1)
 233                         sync += self.r_instr_fault.eq(1)
 234                         comb += maddr.eq(self.maddr)
 235                         sync += self.state.eq(State.MMU_LOOKUP)
 236                     with m.Else():
 237                         sync += self.r_instr_fault.eq(0)
 238                 with m.Else():
 239                     sync += ldst_r.eq(0)
 240
 241             # waiting for completion
 242             with m.Case(State.ACK_WAIT):
 243                 comb += Display("MMUTEST: ACK_WAIT")
 244                 comb += self.busy.eq(~exc.happened)
 245
 246                 with m.If(d_in.error):
 247                     # cache error is not necessarily "final", it could
 248                     # be that it was just a TLB miss
 249                     with m.If(d_in.cache_paradox):
 250                         comb += exception.eq(1)
 251                         sync += self.state.eq(State.IDLE)
 252                         sync += ldst_r.eq(0)
 253                         sync += Display("cache error -> update dsisr")
 254                         sync += self.dsisr[63 - 38].eq(~self.load)
 255                         # XXX there is no architected bit for this
 256                         # (probably should be a machine check in fact)
 257                         sync += self.dsisr[63 - 35].eq(d_in.cache_paradox)
 258
 259                     with m.Else():
 260                         # Look up the translation for TLB miss
 261                         # and also for permission error and RC error
 262                         # in case the PTE has been updated.
 263                         comb += mmureq.eq(1)
 264                         sync += self.state.eq(State.MMU_LOOKUP)
 265                 with m.If(d_in.valid):
 266                     m.d.comb += self.done.eq(~mmureq) # done if not doing MMU
 267                     with m.If(self.done):
 268                         sync += Display("ACK_WAIT, done %x", self.addr)
 269                     sync += self.state.eq(State.IDLE)
 270                     sync += ldst_r.eq(0)
 271                     with m.If(self.load):
 272                         m.d.comb += self.load_data.eq(d_in.data)
 273
 274             # waiting here for the MMU TLB lookup to complete.
 275             # either re-try the dcache lookup or throw MMU exception
 276             with m.Case(State.MMU_LOOKUP):
 277                 comb += self.busy.eq(~exception)
 278                 with m.If(m_in.done):
 279                     with m.If(~self.r_instr_fault):
 280                         sync += Display("MMU_LOOKUP, done %x -> %x",
 281                                         self.addr, d_out.addr)
 282                         # retry the request now that the MMU has
 283                         # installed a TLB entry, if not exception raised
 284                         m.d.comb += self.d_out.valid.eq(~exception)
 285                         sync += self.state.eq(State.ACK_WAIT)
 286                         sync += ldst_r.eq(0)
 287                     with m.Else():
 288                         sync += self.state.eq(State.IDLE)
 289                         sync += self.r_instr_fault.eq(0)
 290                         comb += self.done.eq(1)
 291
 292                 with m.If(m_in.err):
 293                     # MMU RADIX exception thrown. XXX
 294                     # TODO: critical that the write here has to
 295                     # notify the MMU FSM of the change to dsisr
 296                     comb += exception.eq(1)
 297                     comb += self.done.eq(1)
 298                     sync += Display("MMU RADIX exception thrown")
 299                     sync += Display("TODO: notify MMU of change to dsisr")
 300                     sync += self.dsisr[63 - 33].eq(m_in.invalid)
 301                     sync += self.dsisr[63 - 36].eq(m_in.perm_error) # noexec
 302                     sync += self.dsisr[63 - 38].eq(~self.load)
 303                     sync += self.dsisr[63 - 44].eq(m_in.badtree)
 304                     sync += self.dsisr[63 - 45].eq(m_in.rc_error)
 305                     sync += self.state.eq(State.IDLE)
 306                     # exception thrown, clear out instruction fault state
 307                     sync += self.r_instr_fault.eq(0)
 308
 309             with m.Case(State.TLBIE_WAIT):
 310                 pass
 311
 312         # MMU FSM communicating a request to update DSISR or DAR (OP_MTSPR)
 313         with m.If(self.mmu_set_spr):
 314             with m.If(self.mmu_set_dsisr):
 315                 sync += self.dsisr.eq(self.sprval_in)
 316             with m.If(self.mmu_set_dar):
 317                 sync += self.dar.eq(self.sprval_in)
 318
 319         # hmmm, alignment occurs in set_rd_addr/set_wr_addr, note exception
 320         with m.If(self.align_intr):
 321             comb += exc.happened.eq(1)
 322         # check for updating DAR
 323         with m.If(exception):
 324             sync += Display("exception %x", self.addr)
 325             # alignment error: store address in DAR
 326             with m.If(self.align_intr):
 327                 sync += Display("alignment error: addr in DAR %x", self.addr)
 328                 sync += self.dar.eq(self.addr)
 329             with m.Elif(~self.r_instr_fault):
 330                 sync += Display("not instr fault, addr in DAR %x", self.addr)
 331                 sync += self.dar.eq(self.addr)
 332
 333         # when done or exception, return to idle state
 334         with m.If(self.done | exception):
 335             sync += self.state.eq(State.IDLE)
 336             comb += self.busy.eq(0)
 337
 338         # happened, alignment, instr_fault, invalid.
 339         # note that all of these flow through - eventually to the TRAP
 340         # pipeline, via PowerDecoder2.
 341         comb += self.align_intr.eq(self.req.align_intr)
 342         comb += exc.invalid.eq(m_in.invalid)
 343         comb += exc.alignment.eq(self.align_intr)
 344         comb += exc.instr_fault.eq(self.r_instr_fault)
 345         # badtree, perm_error, rc_error, segment_fault
 346         comb += exc.badtree.eq(m_in.badtree)
 347         comb += exc.perm_error.eq(m_in.perm_error)
 348         comb += exc.rc_error.eq(m_in.rc_error)
 349         comb += exc.segment_fault.eq(m_in.segerr)
 350
 351         # TODO, connect dcache wb_in/wb_out to "standard" nmigen Wishbone bus
 352         comb += dbus.adr.eq(dcache.bus.adr)
 353         comb += dbus.dat_w.eq(dcache.bus.dat_w)
 354         comb += dbus.sel.eq(dcache.bus.sel)
 355         comb += dbus.cyc.eq(dcache.bus.cyc)
 356         comb += dbus.stb.eq(dcache.bus.stb)
 357         comb += dbus.we.eq(dcache.bus.we)
 358
 359         comb += dcache.bus.dat_r.eq(dbus.dat_r)
 360         comb += dcache.bus.ack.eq(dbus.ack)
 361         if hasattr(dbus, "stall"):
 362             comb += dcache.bus.stall.eq(dbus.stall)
 363
 364         # update out d data when flag set
 365         with m.If(self.d_w_valid):
 366             m.d.sync += d_out.data.eq(self.store_data)
 367         #with m.Else():
 368         #    m.d.sync += d_out.data.eq(0)
 369         # unit test passes with that change
 370
 371         # this must move into the FSM, conditionally noticing that
 372         # the "blip" comes from self.d_validblip.
 373         # task 1: look up in dcache
 374         # task 2: if dcache fails, look up in MMU.
 375         # do **NOT** confuse the two.
 376         with m.If(self.d_validblip):
 377             m.d.comb += self.d_out.valid.eq(~exc.happened)
 378             m.d.comb += d_out.load.eq(self.req.load)
 379             m.d.comb += d_out.byte_sel.eq(self.req.byte_sel)
 380             m.d.comb += self.addr.eq(self.req.addr)
 381             m.d.comb += d_out.nc.eq(self.req.nc)
 382             m.d.comb += d_out.priv_mode.eq(self.req.priv_mode)
 383             m.d.comb += d_out.virt_mode.eq(self.req.virt_mode)
 384             #m.d.comb += Display("validblip dcbz=%i addr=%x",
 385             #self.req.dcbz,self.req.addr)
 386             m.d.comb += d_out.dcbz.eq(self.req.dcbz)
 387         with m.Else():
 388             m.d.comb += d_out.load.eq(ldst_r.load)
 389             m.d.comb += d_out.byte_sel.eq(ldst_r.byte_sel)
 390             m.d.comb += self.addr.eq(ldst_r.addr)
 391             m.d.comb += d_out.nc.eq(ldst_r.nc)
 392             m.d.comb += d_out.priv_mode.eq(ldst_r.priv_mode)
 393             m.d.comb += d_out.virt_mode.eq(ldst_r.virt_mode)
 394             #m.d.comb += Display("no_validblip dcbz=%i addr=%x",
 395             #ldst_r.dcbz,ldst_r.addr)
 396             m.d.comb += d_out.dcbz.eq(ldst_r.dcbz)
 397
 398         # XXX these should be possible to remove but for some reason
 399         # cannot be... yet. TODO, investigate
 400         m.d.comb += self.load_data.eq(d_in.data)
 401         m.d.comb += d_out.addr.eq(self.addr)
 402
 403         # Update outputs to MMU
 404         m.d.comb += m_out.valid.eq(mmureq)
 405         m.d.comb += m_out.iside.eq(self.instr_fault)
 406         m.d.comb += m_out.load.eq(ldst_r.load)
 407         m.d.comb += m_out.priv.eq(self.priv_mode)
 408         # m_out.priv <= r.priv_mode; TODO
 409         m.d.comb += m_out.tlbie.eq(self.tlbie)
 410         # m_out.mtspr <= mmu_mtspr; # TODO
 411         # m_out.sprn <= sprn; # TODO
 412         m.d.comb += m_out.addr.eq(maddr)
 413         # m_out.slbia <= l_in.insn(7); # TODO: no idea what this is
 414         # m_out.rs <= l_in.data; # nope, probably not needed, TODO investigate
 415
 416         return m
 417
 418     def ports(self):
 419         yield from super().ports()
 420         # TODO: memory ports
 421
 422
 423 class TestSRAMLoadStore1(LoadStore1):
 424     def __init__(self, pspec):
 425         super().__init__(pspec)
 426         pspec = self.pspec
 427         # small 32-entry Memory
 428         if (hasattr(pspec, "dmem_test_depth") and
 429                 isinstance(pspec.dmem_test_depth, int)):
 430             depth = pspec.dmem_test_depth
 431         else:
 432             depth = 32
 433         print("TestSRAMBareLoadStoreUnit depth", depth)
 434
 435         self.mem = Memory(width=pspec.reg_wid, depth=depth)
 436
 437     def elaborate(self, platform):
 438         m = super().elaborate(platform)
 439         comb = m.d.comb
 440         m.submodules.sram = sram = SRAM(memory=self.mem, granularity=8,
 441                                         features={'cti', 'bte', 'err'})
 442         dbus = self.dbus
 443
 444         # directly connect the wishbone bus of LoadStoreUnitInterface to SRAM
 445         # note: SRAM is a target (slave), dbus is initiator (master)
 446         fanouts = ['dat_w', 'sel', 'cyc', 'stb', 'we', 'cti', 'bte']
 447         fanins = ['dat_r', 'ack', 'err']
 448         for fanout in fanouts:
 449             print("fanout", fanout, getattr(sram.bus, fanout).shape(),
 450                   getattr(dbus, fanout).shape())
 451             comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
 452             comb += getattr(sram.bus, fanout).eq(getattr(dbus, fanout))
 453         for fanin in fanins:
 454             comb += getattr(dbus, fanin).eq(getattr(sram.bus, fanin))
 455         # connect address
 456         comb += sram.bus.adr.eq(dbus.adr)
 457
 458         return m
 459