gram/core/multiplexer.py

   1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
   4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
   5 # License: BSD
   6
   7 """LiteDRAM Multiplexer."""
   8
   9 import math
  10
  11 from nmigen import *
  12 from nmigen.asserts import Assert, Assume
  13 from nmigen.lib.scheduler import RoundRobin
  14
  15 from gram.common import *
  16 import gram.stream as stream
  17 from gram.compat import delayed_enter
  18
  19 __ALL__ = ["Multiplexer"]
  20
  21 class _CommandChooser(Elaboratable):
  22     """Arbitrates between requests, filtering them based on their type
  23
  24     Uses RoundRobin to choose current request, filters requests based on
  25     `want_*` signals.
  26
  27     Parameters
  28     ----------
  29     requests : [Endpoint(cmd_request_rw_layout), ...]
  30         Request streams to consider for arbitration
  31
  32     Attributes
  33     ----------
  34     want_reads : Signal, in
  35         Consider read requests
  36     want_writes : Signal, in
  37         Consider write requests
  38     want_cmds : Signal, in
  39         Consider command requests (without ACT)
  40     want_activates : Signal, in
  41         Also consider ACT commands
  42     cmd : Endpoint(cmd_request_rw_layout)
  43         Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
  44     """
  45
  46     def __init__(self, requests):
  47         self.want_reads = Signal()
  48         self.want_writes = Signal()
  49         self.want_cmds = Signal()
  50         self.want_activates = Signal()
  51
  52         self._requests = requests
  53         a = len(requests[0].a)
  54         ba = len(requests[0].ba)
  55
  56         # cas/ras/we are 0 when valid is inactive
  57         self.cmd = stream.Endpoint(cmd_request_rw_layout(a, ba))
  58         self.ready = Signal(len(requests))
  59
  60     def elaborate(self, platform):
  61         m = Module()
  62
  63         n = len(self._requests)
  64
  65         valids = Signal(n)
  66         for i, request in enumerate(self._requests):
  67             is_act_cmd = request.ras & ~request.cas & ~request.we
  68             command = request.is_cmd & self.want_cmds & (~is_act_cmd | self.want_activates)
  69             read = request.is_read == self.want_reads
  70             write = request.is_write == self.want_writes
  71             m.d.comb += valids[i].eq(request.valid & (command | (read & write)))
  72
  73         # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
  74         # command is selected when cmd.ready goes high.
  75         m.submodules.arbiter = arbiter = EnableInserter(self.cmd.ready | ~self.cmd.valid)(RoundRobin(count=n))
  76         choices = Array(valids[i] for i in range(n))
  77         m.d.comb += [
  78             arbiter.requests.eq(valids),
  79             self.cmd.valid.eq(choices[arbiter.grant])
  80         ]
  81
  82         for name in ["a", "ba", "is_read", "is_write", "is_cmd"]:
  83             choices = Array(getattr(req, name) for req in self._requests)
  84             m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  85
  86         for name in ["cas", "ras", "we"]:
  87             # we should only assert those signals when valid is 1
  88             choices = Array(getattr(req, name) for req in self._requests)
  89             with m.If(self.cmd.valid):
  90                 m.d.comb += getattr(self.cmd, name).eq(choices[arbiter.grant])
  91
  92         for i, request in enumerate(self._requests):
  93             with m.If(self.cmd.valid & self.cmd.ready & (arbiter.grant == i)):
  94                 m.d.comb += self.ready[i].eq(1)
  95
  96         return m
  97
  98     # helpers
  99     def accept(self):
 100         return self.cmd.valid & self.cmd.ready
 101
 102     def activate(self):
 103         return self.cmd.ras & ~self.cmd.cas & ~self.cmd.we
 104
 105     def write(self):
 106         return self.cmd.is_write
 107
 108     def read(self):
 109         return self.cmd.is_read
 110
 111 # _Steerer -----------------------------------------------------------------------------------------
 112
 113
 114 (STEER_NOP, STEER_CMD, STEER_REQ, STEER_REFRESH) = range(4)
 115
 116
 117 class _Steerer(Elaboratable):
 118     """Connects selected request to DFI interface
 119
 120     cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
 121     Rank bits are decoded and used to drive cs_n in multi-rank systems,
 122     STEER_REFRESH always enables all ranks.
 123
 124     Parameters
 125     ----------
 126     commands : [Endpoint(cmd_request_rw_layout), ...]
 127         Command streams to choose from. Must be of len=4 in the order:
 128             NOP, CMD, REQ, REFRESH
 129         NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
 130         always considered invalid (because of lack of the `valid` attribute).
 131     dfi : dfi.Interface
 132         DFI interface connected to PHY
 133
 134     Attributes
 135     ----------
 136     sel : [Signal(range(len(commands))), ...], in
 137         Signals for selecting which request gets connected to the corresponding
 138         DFI phase. The signals should take one of the values from STEER_* to
 139         select given source.
 140     """
 141
 142     def __init__(self, commands, dfi):
 143         if len(commands) != 4:
 144             raise ValueError("Commands is not the right size")
 145
 146         self._commands = commands
 147         self._dfi = dfi
 148         self.sel = [Signal(range(len(commands))) for i in range(len(dfi.phases))]
 149
 150     def elaborate(self, platform):
 151         m = Module()
 152
 153         commands = self._commands
 154         dfi = self._dfi
 155
 156         def valid_and(cmd, attr):
 157             if not hasattr(cmd, "valid"):
 158                 return 0
 159             else:
 160                 return cmd.valid & cmd.ready & getattr(cmd, attr)
 161
 162         for i, (phase, sel) in enumerate(zip(dfi.phases, self.sel)):
 163             nranks = len(phase.cs)
 164             rankbits = log2_int(nranks)
 165             if hasattr(phase, "reset"):
 166                 m.d.comb += phase.reset.eq(0)
 167             m.d.comb += phase.clk_en.eq(Repl(Signal(reset=1), nranks))
 168             if hasattr(phase, "odt"):
 169                 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
 170                 m.d.comb += phase.odt.eq(Repl(Signal(reset=1), nranks))
 171             if rankbits:
 172                 rank_decoder = Decoder(nranks)
 173                 m.submodules += rank_decoder
 174                 m.d.comb += rank_decoder.i.eq(
 175                     (Array(cmd.ba[-rankbits:] for cmd in commands)[sel]))
 176                 if i == 0:  # Select all ranks on refresh.
 177                     with m.If(sel == STEER_REFRESH):
 178                         m.d.sync += phase.cs.eq(1)
 179                     with m.Else():
 180                         m.d.sync += phase.cs.eq(rank_decoder.o)
 181                 else:
 182                     m.d.sync += phase.cs.eq(rank_decoder.o)
 183                 m.d.sync += phase.bank.eq(Array(cmd.ba[:-rankbits]
 184                                                 for cmd in commands)[sel])
 185             else:
 186                 m.d.sync += [
 187                     phase.cs.eq(1),
 188                     phase.bank.eq(Array(cmd.ba[:] for cmd in commands)[sel]),
 189                 ]
 190
 191             m.d.sync += [
 192                 phase.address.eq(Array(cmd.a for cmd in commands)[sel]),
 193                 phase.cas.eq(Array(valid_and(cmd, "cas") for cmd in commands)[sel]),
 194                 phase.ras.eq(Array(valid_and(cmd, "ras") for cmd in commands)[sel]),
 195                 phase.we.eq(Array(valid_and(cmd, "we") for cmd in commands)[sel])
 196             ]
 197
 198             rddata_ens = Array(valid_and(cmd, "is_read") for cmd in commands)
 199             wrdata_ens = Array(valid_and(cmd, "is_write") for cmd in commands)
 200             m.d.sync += [
 201                 phase.rddata_en.eq(rddata_ens[sel]),
 202                 phase.wrdata_en.eq(wrdata_ens[sel])
 203             ]
 204
 205         return m
 206
 207 class _AntiStarvation(Elaboratable):
 208     def __init__(self, timeout):
 209         if timeout < 2:
 210             raise ValueError("Timeout values under 2 are not currently supported")
 211
 212         self.en = Signal()
 213         self.max_time = Signal(reset=1)
 214         self._timeout = timeout
 215
 216     def elaborate(self, platform):
 217         m = Module()
 218
 219         if self._timeout > 0:
 220             time = Signal(range(self._timeout))
 221             with m.If(~self.en):
 222                 m.d.sync += [
 223                     time.eq(self._timeout-1),
 224                     self.max_time.eq(0),
 225                 ]
 226             with m.Elif(time != 0):
 227                 m.d.sync += time.eq(time-1)
 228                 with m.If(time == 1):
 229                     m.d.sync += self.max_time.eq(1)
 230                 with m.Else():
 231                     m.d.sync += self.max_time.eq(0)
 232         else:
 233             m.d.comb += self.max_time.eq(0)
 234
 235         if platform == "formal" and self._timeout > 0:
 236             m.d.comb += Assert(self.max_time == (time == 0))
 237
 238         return m
 239
 240 class Multiplexer(Elaboratable):
 241     """Multplexes requets from BankMachines to DFI
 242
 243     This module multiplexes requests from BankMachines (and Refresher) and
 244     connects them to DFI. Refresh commands are coordinated between the Refresher
 245     and BankMachines to ensure there are no conflicts. Enforces required timings
 246     between commands (some timings are enforced by BankMachines).
 247
 248     Parameters
 249     ----------
 250     settings : ControllerSettings
 251         Controller settings (with .phy, .geom and .timing settings)
 252     bank_machines : [BankMachine, ...]
 253         Bank machines that generate command requests to the Multiplexer
 254     refresher : Refresher
 255         Generates REFRESH command requests
 256     dfi : dfi.Interface
 257         DFI connected to the PHY
 258     interface : LiteDRAMInterface
 259         Data interface connected directly to LiteDRAMCrossbar
 260     """
 261
 262     def __init__(self,
 263                  settings,
 264                  bank_machines,
 265                  refresher,
 266                  dfi,
 267                  interface):
 268         assert(settings.phy.nphases == len(dfi.phases))
 269         self._settings = settings
 270         self._bank_machines = bank_machines
 271         self._refresher = refresher
 272         self._dfi = dfi
 273         self._interface = interface
 274
 275     def elaborate(self, platform):
 276         m = Module()
 277
 278         settings = self._settings
 279         bank_machines = self._bank_machines
 280         refresher = self._refresher
 281         dfi = self._dfi
 282         interface = self._interface
 283
 284         ras_allowed = Signal(reset=1)
 285         cas_allowed = Signal(reset=1)
 286
 287         # Command choosing -------------------------------------------------------------------------
 288         requests = [bm.cmd for bm in bank_machines]
 289         m.submodules.choose_cmd = choose_cmd = _CommandChooser(requests)
 290         m.submodules.choose_req = choose_req = _CommandChooser(requests)
 291         for i, request in enumerate(requests):
 292             m.d.comb += request.ready.eq(
 293                 choose_cmd.ready[i] | choose_req.ready[i])
 294         if settings.phy.nphases == 1:
 295             # When only 1 phase, use choose_req for all requests
 296             choose_cmd = choose_req
 297             m.d.comb += choose_req.want_cmds.eq(1)
 298             m.d.comb += choose_req.want_activates.eq(ras_allowed)
 299
 300         # Command steering -------------------------------------------------------------------------
 301         nop = Record(cmd_request_layout(settings.geom.addressbits,
 302                                         log2_int(len(bank_machines))))
 303         # nop must be 1st
 304         commands = [nop, choose_cmd.cmd, choose_req.cmd, refresher.cmd]
 305         m.submodules.steerer = steerer = _Steerer(commands, dfi)
 306
 307         # tRRD timing (Row to Row delay) -----------------------------------------------------------
 308         m.submodules.trrdcon = trrdcon = tXXDController(settings.timing.tRRD)
 309         m.d.comb += trrdcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 310
 311         # tFAW timing (Four Activate Window) -------------------------------------------------------
 312         m.submodules.tfawcon = tfawcon = tFAWController(settings.timing.tFAW)
 313         m.d.comb += tfawcon.valid.eq(choose_cmd.accept() & choose_cmd.activate())
 314
 315         # RAS control ------------------------------------------------------------------------------
 316         m.d.comb += ras_allowed.eq(trrdcon.ready & tfawcon.ready)
 317
 318         # tCCD timing (Column to Column delay) -----------------------------------------------------
 319         m.submodules.tccdcon = tccdcon = tXXDController(settings.timing.tCCD)
 320         m.d.comb += tccdcon.valid.eq(choose_req.accept() & (choose_req.write() | choose_req.read()))
 321
 322         # CAS control ------------------------------------------------------------------------------
 323         m.d.comb += cas_allowed.eq(tccdcon.ready)
 324
 325         # tWTR timing (Write to Read delay) --------------------------------------------------------
 326         write_latency = math.ceil(settings.phy.cwl / settings.phy.nphases)
 327         m.submodules.twtrcon = twtrcon = tXXDController(
 328             settings.timing.tWTR + write_latency +
 329             # tCCD must be added since tWTR begins after the transfer is complete
 330             settings.timing.tCCD if settings.timing.tCCD is not None else 0)
 331         m.d.comb += twtrcon.valid.eq(choose_req.accept() & choose_req.write())
 332
 333         # Read/write turnaround --------------------------------------------------------------------
 334         reads = Signal(len(requests))
 335         m.d.comb += reads.eq(Cat([req.valid & req.is_read for req in requests]))
 336         writes = Signal(len(requests))
 337         m.d.comb += writes.eq(Cat([req.valid & req.is_write for req in requests]))
 338
 339         # Anti Starvation --------------------------------------------------------------------------
 340         m.submodules.read_antistarvation = read_antistarvation = _AntiStarvation(settings.read_time)
 341         m.submodules.write_antistarvation = write_antistarvation = _AntiStarvation(settings.write_time)
 342
 343         # Refresh ----------------------------------------------------------------------------------
 344         m.d.comb += [bm.refresh_req.eq(refresher.cmd.valid) for bm in bank_machines]
 345         bm_refresh_gnts = Signal(len(bank_machines))
 346         m.d.comb += bm_refresh_gnts.eq(Cat([bm.refresh_gnt for bm in bank_machines]))
 347
 348         # Datapath ---------------------------------------------------------------------------------
 349         all_rddata = [p.rddata for p in dfi.phases]
 350         all_wrdata = [p.wrdata for p in dfi.phases]
 351         all_wrdata_mask = [p.wrdata_mask for p in dfi.phases]
 352         m.d.comb += [
 353             interface.rdata.eq(Cat(*all_rddata)),
 354             Cat(*all_wrdata).eq(interface.wdata),
 355             Cat(*all_wrdata_mask).eq(~interface.wdata_we)
 356         ]
 357
 358         # Control FSM ------------------------------------------------------------------------------
 359         with m.FSM():
 360             with m.State("Read"):
 361                 m.d.comb += [
 362                     read_antistarvation.en.eq(1),
 363                     choose_req.want_reads.eq(1),
 364                 ]
 365
 366                 for i in range(settings.phy.nphases):
 367                     if i == settings.phy.rdphase:
 368                         m.d.comb += steerer.sel[i].eq(STEER_REQ)
 369                     elif i == settings.phy.rdcmdphase:
 370                         m.d.comb += steerer.sel[i].eq(STEER_CMD)
 371
 372                 with m.If(settings.phy.nphases == 1):
 373                     m.d.comb += choose_req.cmd.ready.eq(cas_allowed & (~choose_req.activate() | ras_allowed))
 374                 with m.Else():
 375                     m.d.comb += [
 376                         choose_cmd.want_activates.eq(ras_allowed),
 377                         choose_cmd.cmd.ready.eq(~choose_cmd.activate() | ras_allowed),
 378                         choose_req.cmd.ready.eq(cas_allowed),
 379                     ]
 380
 381                 with m.If(writes.any()):
 382                     # TODO: switch only after several cycles of ~reads.any()?
 383                     with m.If(~reads.any() | read_antistarvation.max_time):
 384                         m.next = "RTW"
 385
 386                 with m.If(bm_refresh_gnts.all()):
 387                     m.next = "Refresh"
 388
 389             with m.State("Write"):
 390                 m.d.comb += [
 391                     write_antistarvation.en.eq(1),
 392                     choose_req.want_writes.eq(1),
 393                 ]
 394
 395                 for i in range(settings.phy.nphases):
 396                     if i == settings.phy.wrphase:
 397                         m.d.comb += steerer.sel[i].eq(STEER_REQ)
 398                     elif i == settings.phy.wrcmdphase:
 399                         m.d.comb += steerer.sel[i].eq(STEER_CMD)
 400
 401                 with m.If(settings.phy.nphases == 1):
 402                     m.d.comb += choose_req.cmd.ready.eq(
 403                         cas_allowed & (~choose_req.activate() | ras_allowed))
 404                 with m.Else():
 405                     m.d.comb += [
 406                         choose_cmd.want_activates.eq(ras_allowed),
 407                         choose_cmd.cmd.ready.eq(~choose_cmd.activate() | ras_allowed),
 408                         choose_req.cmd.ready.eq(cas_allowed),
 409                     ]
 410
 411                 with m.If(reads.any()):
 412                     with m.If(~writes.any() | write_antistarvation.max_time):
 413                         m.next = "WTR"
 414
 415                 with m.If(bm_refresh_gnts.all()):
 416                     m.next = "Refresh"
 417
 418             with m.State("Refresh"):
 419                 m.d.comb += [
 420                     steerer.sel[0].eq(STEER_REFRESH),
 421                     refresher.cmd.ready.eq(1),
 422                 ]
 423                 with m.If(refresher.cmd.last):
 424                     m.next = "Read"
 425
 426             with m.State("WTR"):
 427                 with m.If(twtrcon.ready):
 428                     m.next = "Read"
 429
 430             # TODO: reduce this, actual limit is around (cl+1)/nphases
 431             delayed_enter(m, "RTW", "Write", settings.phy.read_latency-1)
 432
 433         return m