gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10
  11 from nmigen import *
  12 from nmigen.asserts import Assert, Assume
  13 from nmigen.lib.scheduler import RoundRobin
  14
  15 from gram.common import *
  16 import gram.stream as stream
  17 from gram.compat import delayed_enter
  18
  19 __ALL__ = ["Multiplexer"]
  20
  21 class _CommandChooser(Elaboratable):
  22     """Arbitrates between requests, filtering them based on their type
  23
  24     Uses RoundRobin to choose current request, filters requests based on
  25     `want_*` signals.
  26
  27     Parameters
  28     ----------
  29     requests : [Endpoint(cmd_request_rw_layout), ...]
  30         Request streams to consider for arbitration
  31
  32     Attributes
  33     ----------
  34     want_reads : Signal, in
  35         Consider read requests
  36     want_writes : Signal, in
  37         Consider write requests
  38     want_cmds : Signal, in
  39         Consider command requests (without ACT)
  40     want_activates : Signal, in
  41         Also consider ACT commands
  42     cmd : Endpoint(cmd_request_rw_layout)
  43         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  44     """
  45
  46     def __init__(self, requests):
  47         self.want_reads = Signal()
  48         self.want_writes = Signal()
  49         self.want_cmds = Signal()
  50         self.want_activates = Signal()
  51
  52         self._requests = requests
  53         a = len(requests[0].a)
  54         ba = len(requests[0].ba)
  55
  56         # cas/ras/we are 0 when valid is inactive
  57         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  58         self.ready = Signal(len(requests))
  59
  60     def elaborate(self, platform):
  61         m = Module()
  62
  63         n = len(self._requests)
  64
  65         valids = Signal(n)
  66         for i, request in enumerate(self._requests):
  67             is_act_cmd = request.ras & ~request.cas & ~request.we
  68             command = request.is_cmd & self.want_cmds & (~is_act_cmd | self.want_activates)
  69             read = request.is_read == self.want_reads
  70             write = request.is_write == self.want_writes
  71             m.d.comb += valids[i].eq(request.valid & (command | (read & write)))
  72
  73         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
  74         # command is selected when cmd.ready goes high.
  75         m.submodules.arbiter = arbiter = EnableInserter(self.cmd.ready | ~self.cmd.valid)(RoundRobin(count=n))
  76         choices = Array(valids[i] for i in range(n))
  77         m.d.comb += [
  78             arbiter.requests.eq(valids),
  79             self.cmd.valid.eq(choices[arbiter.grant])
  80         ]
  81
  82         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  83             choices = Array(getattr(req, name) for req in self._requests)
  84             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  85
  86         for name in ["cas", "ras", "we"]:
  87             # we should only assert those signals when valid is 1
  88             choices = Array(getattr(req, name) for req in self._requests)
  89             with m.If(self.cmd.valid):
  90                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  91
  92         for i, request in enumerate(self._requests):
  93             with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  94                 m.d.comb += self.ready[i].eq(1)
  95
  96         return m
  97
  98     # helpers
  99     def accept(self):
 100         return self.cmd.valid & self.cmd.ready
 101
 102     def activate(self):
 103         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 104
 105     def write(self):
 106         return self.cmd.is_write
 107
 108     def read(self):
 109         return self.cmd.is_read
 110
 111 # _Steerer -----------------------------------------------------------------------------------------
 112
 113
 114 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 115
 116
 117 class _Steerer(Elaboratable):
 118     """Connects selected request to DFI interface
 119
 120     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 121     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 122     STEER_REFRESH always enables all ranks.
 123
 124     Parameters
 125     ----------
 126     commands : [Endpoint(cmd_request_rw_layout), ...]
 127         Command streams to choose from. Must be of len=4 in the order:
 128             NOP, CMD, REQ, REFRESH
 129         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 130         always considered invalid (because of lack of the `valid` attribute).
 131     dfi : dfi.Interface
 132         DFI interface connected to PHY
 133
 134     Attributes
 135     ----------
 136     sel : [Signal(range(len(commands))), ...], in
 137         Signals for selecting which request gets connected to the corresponding
 138         DFI phase. The signals should take one of the values from STEER_* to
 139         select given source.
 140     """
 141
 142     def __init__(self, commands, dfi):
 143         if len(commands) != 4:
 144             raise ValueError("Commands is not the right size")
 145
 146         self.commands = commands
 147         self.dfi = dfi
 148         self.sel = [Signal(range(len(commands))) for i in range(len(dfi.phases))]
 149
 150     def elaborate(self, platform):
 151         m = Module()
 152
 153         def valid_and(cmd, attr):
 154             if not hasattr(cmd, "valid"):
 155                 return 0
 156             else:
 157                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 158
 159         for i, (phase, sel) in enumerate(zip(self.dfi.phases, self.sel)):
 160             nranks = len(phase.cs_n)
 161             rankbits = log2_int(nranks)
 162             if hasattr(phase, "reset_n"):
 163                 m.d.comb += phase.reset_n.eq(1)
 164             m.d.comb += phase.clk_en.eq(Repl(1, nranks))
 165             if hasattr(phase, "odt"):
 166                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 167                 m.d.comb += phase.odt.eq(Repl(1, nranks))
 168             if rankbits:
 169                 rank_decoder = Decoder(nranks)
 170                 m.submodules += rank_decoder
 171                 m.d.comb += rank_decoder.i.eq((Array(cmd.ba[-rankbits:] for cmd in self.commands)[sel]))
 172                 if i == 0:  # Select all ranks on refresh.
 173                     with m.If(sel == STEER_REFRESH):
 174                         m.d.sync += phase.cs_n.eq(0)
 175                     with m.Else():
 176                         m.d.sync += phase.cs_n.eq(rank_decoder.o)
 177                 else:
 178                     m.d.sync += phase.cs_n.eq(rank_decoder.o)
 179                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits] for cmd in self.commands)[sel])
 180             else:
 181                 m.d.sync += [
 182                     phase.cs_n.eq(0),
 183                     phase.bank.eq(Array(cmd.ba for cmd in self.commands)[sel]),
 184                 ]
 185
 186             m.d.sync += [
 187                 phase.address.eq(Array(cmd.a for cmd in self.commands)[sel]),
 188                 phase.cas.eq(Array(valid_and(cmd, "cas") for cmd in self.commands)[sel]),
 189                 phase.ras.eq(Array(valid_and(cmd, "ras") for cmd in self.commands)[sel]),
 190                 phase.we.eq(Array(valid_and(cmd, "we") for cmd in self.commands)[sel])
 191             ]
 192
 193             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in self.commands)
 194             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in self.commands)
 195             m.d.sync += [
 196                 phase.rddata_en.eq(rddata_ens[sel]),
 197                 phase.wrdata_en.eq(wrdata_ens[sel])
 198             ]
 199
 200         return m
 201
 202 class _AntiStarvation(Elaboratable):
 203     def __init__(self, timeout):
 204         if timeout < 2:
 205             raise ValueError("Timeout values under 2 are not currently supported")
 206
 207         self.en = Signal()
 208         self.max_time = Signal(reset=1)
 209         self._timeout = timeout
 210
 211     def elaborate(self, platform):
 212         m = Module()
 213
 214         if self._timeout > 0:
 215             time = Signal(range(self._timeout))
 216             with m.If(~self.en):
 217                 m.d.sync += [
 218                     time.eq(self._timeout-1),
 219                     self.max_time.eq(0),
 220                 ]
 221             with m.Elif(time != 0):
 222                 m.d.sync += time.eq(time-1)
 223                 with m.If(time == 1):
 224                     m.d.sync += self.max_time.eq(1)
 225                 with m.Else():
 226                     m.d.sync += self.max_time.eq(0)
 227         else:
 228             m.d.comb += self.max_time.eq(0)
 229
 230         if platform == "formal" and self._timeout > 0:
 231             m.d.comb += Assert(self.max_time == (time == 0))
 232
 233         return m
 234
 235 class Multiplexer(Elaboratable):
 236     """Multplexes requets from BankMachines to DFI
 237
 238     This module multiplexes requests from BankMachines (and Refresher) and
 239     connects them to DFI. Refresh commands are coordinated between the Refresher
 240     and BankMachines to ensure there are no conflicts. Enforces required timings
 241     between commands (some timings are enforced by BankMachines).
 242
 243     Parameters
 244     ----------
 245     settings : ControllerSettings
 246         Controller settings (with .phy, .geom and .timing settings)
 247     bank_machines : [BankMachine, ...]
 248         Bank machines that generate command requests to the Multiplexer
 249     refresher : Refresher
 250         Generates REFRESH command requests
 251     dfi : dfi.Interface
 252         DFI connected to the PHY
 253     interface : LiteDRAMInterface
 254         Data interface connected directly to LiteDRAMCrossbar
 255     """
 256
 257     def __init__(self,
 258                  settings,
 259                  bank_machines,
 260                  refresher,
 261                  dfi,
 262                  interface):
 263         assert(settings.phy.nphases == len(dfi.phases))
 264         self._settings = settings
 265         self._bank_machines = bank_machines
 266         self._refresher = refresher
 267         self._dfi = dfi
 268         self._interface = interface
 269
 270     def elaborate(self, platform):
 271         m = Module()
 272
 273         settings = self._settings
 274         bank_machines = self._bank_machines
 275         refresher = self._refresher
 276         dfi = self._dfi
 277         interface = self._interface
 278
 279         ras_allowed = Signal(reset=1)
 280         cas_allowed = Signal(reset=1)
 281
 282         # Command choosing -------------------------------------------------------------------------
 283         requests = [bm.cmd for bm in bank_machines]
 284         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 285         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 286         for i, request in enumerate(requests):
 287             m.d.comb += request.ready.eq(choose_cmd.ready[i] | choose_req.ready[i])
 288         if settings.phy.nphases == 1:
 289             # When only 1 phase, use choose_req for all requests
 290             choose_cmd = choose_req
 291             m.d.comb += choose_req.want_cmds.eq(1)
 292             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 293
 294         # Command steering -------------------------------------------------------------------------
 295         nop = Record(cmd_request_layout(settings.geom.addressbits,
 296                                         log2_int(len(bank_machines))))
 297         # nop must be 1st
 298         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 299         m.submodules.steerer = steerer = _Steerer(commands, dfi)
 300
 301         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 302         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 303         m.d.comb += trrdcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 304
 305         # tFAW timing (Four Activate Window) -------------------------------------------------------
 306         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 307         m.d.comb += tfawcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 308
 309         # RAS control ------------------------------------------------------------------------------
 310         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 311
 312         # tCCD timing (Column to Column delay) -----------------------------------------------------
 313         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 314         m.d.comb += tccdcon.valid.eq(choose_req.accept() & (choose_req.write() | choose_req.read()))
 315
 316         # CAS control ------------------------------------------------------------------------------
 317         m.d.comb += cas_allowed.eq(tccdcon.ready)
 318
 319         # tWTR timing (Write to Read delay) --------------------------------------------------------
 320         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 321         m.submodules.twtrcon = twtrcon = tXXDController(
 322             settings.timing.tWTR + write_latency +
 323             # tCCD must be added since tWTR begins after the transfer is complete
 324             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 325         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 326
 327         # Read/write turnaround --------------------------------------------------------------------
 328         reads = Signal(len(requests))
 329         m.d.comb += reads.eq(Cat([(req.valid & req.is_read) for req in requests]))
 330         writes = Signal(len(requests))
 331         m.d.comb += writes.eq(Cat([(req.valid & req.is_write) for req in requests]))
 332
 333         # Anti Starvation --------------------------------------------------------------------------
 334         m.submodules.read_antistarvation = read_antistarvation = _AntiStarvation(settings.read_time)
 335         m.submodules.write_antistarvation = write_antistarvation = _AntiStarvation(settings.write_time)
 336
 337         # Refresh ----------------------------------------------------------------------------------
 338         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid) for bm in bank_machines]
 339         bm_refresh_gnts = Signal(len(bank_machines))
 340         m.d.comb += bm_refresh_gnts.eq(Cat([bm.refresh_gnt for bm in bank_machines]))
 341
 342         # Datapath ---------------------------------------------------------------------------------
 343         all_rddata = [p.rddata for p in dfi.phases]
 344         all_wrdata = [p.wrdata for p in dfi.phases]
 345         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 346         m.d.comb += [
 347             interface.rdata.eq(Cat(*all_rddata)),
 348             Cat(*all_wrdata).eq(interface.wdata),
 349             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 350         ]
 351
 352         # Control FSM ------------------------------------------------------------------------------
 353         with m.FSM():
 354             with m.State("Read"):
 355                 m.d.comb += [
 356                     read_antistarvation.en.eq(1),
 357                     choose_req.want_reads.eq(1),
 358                 ]
 359
 360                 for i in range(settings.phy.nphases):
 361                     if i == settings.phy.rdphase:
 362                         m.d.comb += steerer.sel[i].eq(STEER_REQ)
 363                     elif i == settings.phy.rdcmdphase:
 364                         m.d.comb += steerer.sel[i].eq(STEER_CMD)
 365                     else:
 366                         m.d.comb += steerer.sel[i].eq(STEER_NOP)
 367
 368                 with m.If(settings.phy.nphases == 1):
 369                     m.d.comb += choose_req.cmd.ready.eq(cas_allowed & (~choose_req.activate() | ras_allowed))
 370                 with m.Else():
 371                     m.d.comb += [
 372                         choose_cmd.want_activates.eq(ras_allowed),
 373                         choose_cmd.cmd.ready.eq(~choose_cmd.activate() | ras_allowed),
 374                         choose_req.cmd.ready.eq(cas_allowed),
 375                     ]
 376
 377                 with m.If(writes.any()):
 378                     # TODO: switch only after several cycles of ~reads.any()?
 379                     with m.If(~reads.any() | read_antistarvation.max_time):
 380                         m.next = "RTW"
 381
 382                 with m.If(bm_refresh_gnts.all()):
 383                     m.next = "Refresh"
 384
 385             with m.State("Write"):
 386                 m.d.comb += [
 387                     write_antistarvation.en.eq(1),
 388                     choose_req.want_writes.eq(1),
 389                 ]
 390
 391                 for i in range(settings.phy.nphases):
 392                     if i == settings.phy.wrphase:
 393                         m.d.comb += steerer.sel[i].eq(STEER_REQ)
 394                     elif i == settings.phy.wrcmdphase:
 395                         m.d.comb += steerer.sel[i].eq(STEER_CMD)
 396                     else:
 397                         m.d.comb += steerer.sel[i].eq(STEER_NOP)
 398
 399                 with m.If(settings.phy.nphases == 1):
 400                     m.d.comb += choose_req.cmd.ready.eq(cas_allowed & (~choose_req.activate() | ras_allowed))
 401                 with m.Else():
 402                     m.d.comb += [
 403                         choose_cmd.want_activates.eq(ras_allowed),
 404                         choose_cmd.cmd.ready.eq(~choose_cmd.activate() | ras_allowed),
 405                         choose_req.cmd.ready.eq(cas_allowed),
 406                     ]
 407
 408                 with m.If(reads.any()):
 409                     with m.If(~writes.any() | write_antistarvation.max_time):
 410                         m.next = "WTR"
 411
 412                 with m.If(bm_refresh_gnts.all()):
 413                     m.next = "Refresh"
 414
 415             with m.State("Refresh"):
 416                 m.d.comb += [
 417                     steerer.sel[0].eq(STEER_REFRESH),
 418                     refresher.cmd.ready.eq(1),
 419                 ]
 420                 with m.If(refresher.cmd.last):
 421                     m.next = "Read"
 422
 423             with m.State("WTR"):
 424                 with m.If(twtrcon.ready):
 425                     m.next = "Read"
 426
 427             # TODO: reduce this, actual limit is around (cl+1)/nphases
 428             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 429
 430         return m