gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10
  11 from nmigen import *
  12 from nmigen.asserts import Assert, Assume
  13 from nmigen.lib.scheduler import RoundRobin
  14
  15 from gram.common import *
  16 import gram.stream as stream
  17 from gram.compat import delayed_enter
  18
  19 __ALL__ = ["Multiplexer"]
  20
  21 class _CommandChooser(Elaboratable):
  22     """Arbitrates between requests, filtering them based on their type
  23
  24     Uses RoundRobin to choose current request, filters requests based on
  25     `want_*` signals.
  26
  27     Parameters
  28     ----------
  29     requests : [Endpoint(cmd_request_rw_layout), ...]
  30         Request streams to consider for arbitration
  31
  32     Attributes
  33     ----------
  34     want_reads : Signal, in
  35         Consider read requests
  36     want_writes : Signal, in
  37         Consider write requests
  38     want_cmds : Signal, in
  39         Consider command requests (without ACT)
  40     want_activates : Signal, in
  41         Also consider ACT commands
  42     cmd : Endpoint(cmd_request_rw_layout)
  43         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  44     """
  45
  46     def __init__(self, requests):
  47         self.want_reads = Signal()
  48         self.want_writes = Signal()
  49         self.want_cmds = Signal()
  50         self.want_activates = Signal()
  51
  52         self._requests = requests
  53         a = len(requests[0].a)
  54         ba = len(requests[0].ba)
  55
  56         # cas/ras/we are 0 when valid is inactive
  57         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  58         self.ready = Signal(len(requests))
  59
  60     def elaborate(self, platform):
  61         m = Module()
  62
  63         n = len(self._requests)
  64
  65         valids = Signal(n)
  66         for i, request in enumerate(self._requests):
  67             is_act_cmd = request.ras & ~request.cas & ~request.we
  68             command = request.is_cmd & self.want_cmds & (~is_act_cmd | self.want_activates)
  69             read = request.is_read == self.want_reads
  70             write = request.is_write == self.want_writes
  71             m.d.comb += valids[i].eq(request.valid & (command | (read & write)))
  72
  73         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
  74         # command is selected when cmd.ready goes high.
  75         m.submodules.arbiter = arbiter = EnableInserter(self.cmd.ready | ~self.cmd.valid)(RoundRobin(count=n))
  76         choices = Array(valids[i] for i in range(n))
  77         m.d.comb += [
  78             arbiter.requests.eq(valids),
  79             self.cmd.valid.eq(choices[arbiter.grant])
  80         ]
  81
  82         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  83             choices = Array(getattr(req, name) for req in self._requests)
  84             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  85
  86         for name in ["cas", "ras", "we"]:
  87             # we should only assert those signals when valid is 1
  88             choices = Array(getattr(req, name) for req in self._requests)
  89             with m.If(self.cmd.valid):
  90                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  91
  92         for i, request in enumerate(self._requests):
  93             with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  94                 m.d.comb += self.ready[i].eq(1)
  95
  96         return m
  97
  98     # helpers
  99     def accept(self):
 100         return self.cmd.valid & self.cmd.ready
 101
 102     def activate(self):
 103         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 104
 105     def write(self):
 106         return self.cmd.is_write
 107
 108     def read(self):
 109         return self.cmd.is_read
 110
 111 # _Steerer -----------------------------------------------------------------------------------------
 112
 113
 114 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 115
 116
 117 class _Steerer(Elaboratable):
 118     """Connects selected request to DFI interface
 119
 120     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 121     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 122     STEER_REFRESH always enables all ranks.
 123
 124     Parameters
 125     ----------
 126     commands : [Endpoint(cmd_request_rw_layout), ...]
 127         Command streams to choose from. Must be of len=4 in the order:
 128             NOP, CMD, REQ, REFRESH
 129         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 130         always considered invalid (because of lack of the `valid` attribute).
 131     dfi : dfi.Interface
 132         DFI interface connected to PHY
 133
 134     Attributes
 135     ----------
 136     sel : [Signal(range(len(commands))), ...], in
 137         Signals for selecting which request gets connected to the corresponding
 138         DFI phase. The signals should take one of the values from STEER_* to
 139         select given source.
 140     """
 141
 142     def __init__(self, commands, dfi):
 143         if len(commands) != 4:
 144             raise ValueError("Commands is not the right size")
 145
 146         self._commands = commands
 147         self._dfi = dfi
 148         self.sel = [Signal(range(len(commands))) for i in range(len(dfi.phases))]
 149
 150     def elaborate(self, platform):
 151         m = Module()
 152
 153         commands = self._commands
 154         dfi = self._dfi
 155
 156         def valid_and(cmd, attr):
 157             if not hasattr(cmd, "valid"):
 158                 return 0
 159             else:
 160                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 161
 162         for i, (phase, sel) in enumerate(zip(dfi.phases, self.sel)):
 163             nranks = len(phase.cs)
 164             rankbits = log2_int(nranks)
 165             if hasattr(phase, "reset"):
 166                 m.d.comb += phase.reset.eq(0)
 167             m.d.comb += phase.clk_en.eq(Repl(Signal(reset=1), nranks))
 168             if hasattr(phase, "odt"):
 169                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 170                 m.d.comb += phase.odt.eq(Repl(Signal(reset=1), nranks))
 171             if rankbits:
 172                 rank_decoder = Decoder(nranks)
 173                 m.submodules += rank_decoder
 174                 m.d.comb += rank_decoder.i.eq(
 175                     (Array(cmd.ba[-rankbits:] for cmd in commands)[sel]))
 176                 if i == 0:  # Select all ranks on refresh.
 177                     with m.If(sel == STEER_REFRESH):
 178                         m.d.sync += phase.cs.eq(1)
 179                     with m.Else():
 180                         m.d.sync += phase.cs.eq(rank_decoder.o)
 181                 else:
 182                     m.d.sync += phase.cs.eq(rank_decoder.o)
 183                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits]
 184                                                 for cmd in commands)[sel])
 185             else:
 186                 m.d.sync += [
 187                     phase.cs.eq(1),
 188                     phase.bank.eq(Array(cmd.ba[:] for cmd in commands)[sel]),
 189                 ]
 190
 191             m.d.sync += [
 192                 phase.address.eq(Array(cmd.a for cmd in commands)[sel]),
 193                 phase.cas.eq(Array(valid_and(cmd, "cas") for cmd in commands)[sel]),
 194                 phase.ras.eq(Array(valid_and(cmd, "ras") for cmd in commands)[sel]),
 195                 phase.we.eq(Array(valid_and(cmd, "we") for cmd in commands)[sel])
 196             ]
 197
 198             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in commands)
 199             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in commands)
 200             m.d.sync += [
 201                 phase.rddata_en.eq(rddata_ens[sel]),
 202                 phase.wrdata_en.eq(wrdata_ens[sel])
 203             ]
 204
 205         return m
 206
 207 class _AntiStarvation(Elaboratable):
 208     def __init__(self, timeout):
 209         if timeout < 2:
 210             raise ValueError("Timeout values under 2 are not currently supported")
 211
 212         self.en = Signal()
 213         self.max_time = Signal(reset=1)
 214         self._timeout = timeout
 215
 216     def elaborate(self, platform):
 217         m = Module()
 218
 219         if self._timeout > 0:
 220             time = Signal(range(self._timeout))
 221             with m.If(~self.en):
 222                 m.d.sync += [
 223                     time.eq(self._timeout-1),
 224                     self.max_time.eq(0),
 225                 ]
 226             with m.Elif(time != 0):
 227                 m.d.sync += time.eq(time-1)
 228                 with m.If(time == 1):
 229                     m.d.sync += self.max_time.eq(1)
 230                 with m.Else():
 231                     m.d.sync += self.max_time.eq(0)
 232         else:
 233             m.d.comb += self.max_time.eq(0)
 234
 235         if platform == "formal" and self._timeout > 0:
 236             m.d.comb += Assert(self.max_time == (time == 0))
 237
 238         return m
 239
 240 class Multiplexer(Elaboratable):
 241     """Multplexes requets from BankMachines to DFI
 242
 243     This module multiplexes requests from BankMachines (and Refresher) and
 244     connects them to DFI. Refresh commands are coordinated between the Refresher
 245     and BankMachines to ensure there are no conflicts. Enforces required timings
 246     between commands (some timings are enforced by BankMachines).
 247
 248     Parameters
 249     ----------
 250     settings : ControllerSettings
 251         Controller settings (with .phy, .geom and .timing settings)
 252     bank_machines : [BankMachine, ...]
 253         Bank machines that generate command requests to the Multiplexer
 254     refresher : Refresher
 255         Generates REFRESH command requests
 256     dfi : dfi.Interface
 257         DFI connected to the PHY
 258     interface : LiteDRAMInterface
 259         Data interface connected directly to LiteDRAMCrossbar
 260     """
 261
 262     def __init__(self,
 263                  settings,
 264                  bank_machines,
 265                  refresher,
 266                  dfi,
 267                  interface):
 268         assert(settings.phy.nphases == len(dfi.phases))
 269         self._settings = settings
 270         self._bank_machines = bank_machines
 271         self._refresher = refresher
 272         self._dfi = dfi
 273         self._interface = interface
 274
 275     def elaborate(self, platform):
 276         m = Module()
 277
 278         settings = self._settings
 279         bank_machines = self._bank_machines
 280         refresher = self._refresher
 281         dfi = self._dfi
 282         interface = self._interface
 283
 284         ras_allowed = Signal(reset=1)
 285         cas_allowed = Signal(reset=1)
 286
 287         # Command choosing -------------------------------------------------------------------------
 288         requests = [bm.cmd for bm in bank_machines]
 289         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 290         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 291         for i, request in enumerate(requests):
 292             m.d.comb += request.ready.eq(choose_cmd.ready[i] | choose_req.ready[i])
 293         if settings.phy.nphases == 1:
 294             # When only 1 phase, use choose_req for all requests
 295             choose_cmd = choose_req
 296             m.d.comb += choose_req.want_cmds.eq(1)
 297             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 298
 299         # Command steering -------------------------------------------------------------------------
 300         nop = Record(cmd_request_layout(settings.geom.addressbits,
 301                                         log2_int(len(bank_machines))))
 302         # nop must be 1st
 303         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 304         m.submodules.steerer = steerer = _Steerer(commands, dfi)
 305
 306         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 307         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 308         m.d.comb += trrdcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 309
 310         # tFAW timing (Four Activate Window) -------------------------------------------------------
 311         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 312         m.d.comb += tfawcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 313
 314         # RAS control ------------------------------------------------------------------------------
 315         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 316
 317         # tCCD timing (Column to Column delay) -----------------------------------------------------
 318         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 319         m.d.comb += tccdcon.valid.eq(choose_req.accept() & (choose_req.write() | choose_req.read()))
 320
 321         # CAS control ------------------------------------------------------------------------------
 322         m.d.comb += cas_allowed.eq(tccdcon.ready)
 323
 324         # tWTR timing (Write to Read delay) --------------------------------------------------------
 325         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 326         m.submodules.twtrcon = twtrcon = tXXDController(
 327             settings.timing.tWTR + write_latency +
 328             # tCCD must be added since tWTR begins after the transfer is complete
 329             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 330         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 331
 332         # Read/write turnaround --------------------------------------------------------------------
 333         reads = Signal(len(requests))
 334         m.d.comb += reads.eq(Cat([(req.valid & req.is_read) for req in requests]))
 335         writes = Signal(len(requests))
 336         m.d.comb += writes.eq(Cat([(req.valid & req.is_write) for req in requests]))
 337
 338         # Anti Starvation --------------------------------------------------------------------------
 339         m.submodules.read_antistarvation = read_antistarvation = _AntiStarvation(settings.read_time)
 340         m.submodules.write_antistarvation = write_antistarvation = _AntiStarvation(settings.write_time)
 341
 342         # Refresh ----------------------------------------------------------------------------------
 343         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid) for bm in bank_machines]
 344         bm_refresh_gnts = Signal(len(bank_machines))
 345         m.d.comb += bm_refresh_gnts.eq(Cat([bm.refresh_gnt for bm in bank_machines]))
 346
 347         # Datapath ---------------------------------------------------------------------------------
 348         all_rddata = [p.rddata for p in dfi.phases]
 349         all_wrdata = [p.wrdata for p in dfi.phases]
 350         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 351         m.d.comb += [
 352             interface.rdata.eq(Cat(*all_rddata)),
 353             Cat(*all_wrdata).eq(interface.wdata),
 354             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 355         ]
 356
 357         # Control FSM ------------------------------------------------------------------------------
 358         with m.FSM():
 359             with m.State("Read"):
 360                 m.d.comb += [
 361                     read_antistarvation.en.eq(1),
 362                     choose_req.want_reads.eq(1),
 363                 ]
 364
 365                 for i in range(settings.phy.nphases):
 366                     if i == settings.phy.rdphase:
 367                         m.d.comb += steerer.sel[i].eq(STEER_REQ)
 368                     elif i == settings.phy.rdcmdphase:
 369                         m.d.comb += steerer.sel[i].eq(STEER_CMD)
 370
 371                 with m.If(settings.phy.nphases == 1):
 372                     m.d.comb += choose_req.cmd.ready.eq(cas_allowed & (~choose_req.activate() | ras_allowed))
 373                 with m.Else():
 374                     m.d.comb += [
 375                         choose_cmd.want_activates.eq(ras_allowed),
 376                         choose_cmd.cmd.ready.eq(~choose_cmd.activate() | ras_allowed),
 377                         choose_req.cmd.ready.eq(cas_allowed),
 378                     ]
 379
 380                 with m.If(writes.any()):
 381                     # TODO: switch only after several cycles of ~reads.any()?
 382                     with m.If(~reads.any() | read_antistarvation.max_time):
 383                         m.next = "RTW"
 384
 385                 with m.If(bm_refresh_gnts.all()):
 386                     m.next = "Refresh"
 387
 388             with m.State("Write"):
 389                 m.d.comb += [
 390                     write_antistarvation.en.eq(1),
 391                     choose_req.want_writes.eq(1),
 392                 ]
 393
 394                 for i in range(settings.phy.nphases):
 395                     if i == settings.phy.wrphase:
 396                         m.d.comb += steerer.sel[i].eq(STEER_REQ)
 397                     elif i == settings.phy.wrcmdphase:
 398                         m.d.comb += steerer.sel[i].eq(STEER_CMD)
 399
 400                 with m.If(settings.phy.nphases == 1):
 401                     m.d.comb += choose_req.cmd.ready.eq(cas_allowed & (~choose_req.activate() | ras_allowed))
 402                 with m.Else():
 403                     m.d.comb += [
 404                         choose_cmd.want_activates.eq(ras_allowed),
 405                         choose_cmd.cmd.ready.eq(~choose_cmd.activate() | ras_allowed),
 406                         choose_req.cmd.ready.eq(cas_allowed),
 407                     ]
 408
 409                 with m.If(reads.any()):
 410                     with m.If(~writes.any() | write_antistarvation.max_time):
 411                         m.next = "WTR"
 412
 413                 with m.If(bm_refresh_gnts.all()):
 414                     m.next = "Refresh"
 415
 416             with m.State("Refresh"):
 417                 m.d.comb += [
 418                     steerer.sel[0].eq(STEER_REFRESH),
 419                     refresher.cmd.ready.eq(1),
 420                 ]
 421                 with m.If(refresher.cmd.last):
 422                     m.next = "Read"
 423
 424             with m.State("WTR"):
 425                 with m.If(twtrcon.ready):
 426                     m.next = "Read"
 427
 428             # TODO: reduce this, actual limit is around (cl+1)/nphases
 429             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 430
 431         return m