1 # This file is Copyright (c) 2015 Sebastien Bourdeauducq <sb@m-labs.hk>
2 # This file is Copyright (c) 2016-2019 Florent Kermarrec <florent@enjoy-digital.fr>
3 # This file is Copyright (c) 2018 John Sully <john@csquare.ca>
4 # This file is Copyright (c) 2020 LambdaConcept <contact@lambdaconcept.com>
7 """LiteDRAM Multiplexer."""
12 from nmigen
.asserts
import Assert
, Assume
13 from nmigen
.lib
.scheduler
import RoundRobin
15 from gram
.common
import *
16 import gram
.stream
as stream
17 from gram
.compat
import delayed_enter
19 __ALL__
= ["Multiplexer"]
21 class _CommandChooser(Elaboratable
):
22 """Arbitrates between requests, filtering them based on their type
24 Uses RoundRobin to choose current request, filters requests based on
29 requests : [Endpoint(cmd_request_rw_layout), ...]
30 Request streams to consider for arbitration
34 want_reads : Signal, in
35 Consider read requests
36 want_writes : Signal, in
37 Consider write requests
38 want_cmds : Signal, in
39 Consider command requests (without ACT)
40 want_activates : Signal, in
41 Also consider ACT commands
42 cmd : Endpoint(cmd_request_rw_layout)
43 Currently selected request stream (when ~cmd.valid, cas/ras/we are 0)
46 def __init__(self
, requests
):
47 self
.want_reads
= Signal()
48 self
.want_writes
= Signal()
49 self
.want_cmds
= Signal()
50 self
.want_activates
= Signal()
52 self
._requests
= requests
53 a
= len(requests
[0].a
)
54 ba
= len(requests
[0].ba
)
56 # cas/ras/we are 0 when valid is inactive
57 self
.cmd
= stream
.Endpoint(cmd_request_rw_layout(a
, ba
))
58 self
.ready
= Signal(len(requests
))
60 def elaborate(self
, platform
):
63 n
= len(self
._requests
)
66 for i
, request
in enumerate(self
._requests
):
67 is_act_cmd
= request
.ras
& ~request
.cas
& ~request
.we
68 command
= request
.is_cmd
& self
.want_cmds
& (~is_act_cmd | self
.want_activates
)
69 read
= request
.is_read
== self
.want_reads
70 write
= request
.is_write
== self
.want_writes
71 m
.d
.comb
+= valids
[i
].eq(request
.valid
& (command |
(read
& write
)))
73 # Arbitrate if a command is being accepted or if the command is not valid to ensure a valid
74 # command is selected when cmd.ready goes high.
75 m
.submodules
.arbiter
= arbiter
= EnableInserter(self
.cmd
.ready | ~self
.cmd
.valid
)(RoundRobin(count
=n
))
76 choices
= Array(valids
[i
] for i
in range(n
))
78 arbiter
.requests
.eq(valids
),
79 self
.cmd
.valid
.eq(choices
[arbiter
.grant
])
82 for name
in ["a", "ba", "is_read", "is_write", "is_cmd"]:
83 choices
= Array(getattr(req
, name
) for req
in self
._requests
)
84 m
.d
.comb
+= getattr(self
.cmd
, name
).eq(choices
[arbiter
.grant
])
86 for name
in ["cas", "ras", "we"]:
87 # we should only assert those signals when valid is 1
88 choices
= Array(getattr(req
, name
) for req
in self
._requests
)
89 with m
.If(self
.cmd
.valid
):
90 m
.d
.comb
+= getattr(self
.cmd
, name
).eq(choices
[arbiter
.grant
])
92 for i
, request
in enumerate(self
._requests
):
93 with m
.If(self
.cmd
.valid
& self
.cmd
.ready
& (arbiter
.grant
== i
)):
94 m
.d
.comb
+= self
.ready
[i
].eq(1)
100 return self
.cmd
.valid
& self
.cmd
.ready
103 return self
.cmd
.ras
& ~self
.cmd
.cas
& ~self
.cmd
.we
106 return self
.cmd
.is_write
109 return self
.cmd
.is_read
111 # _Steerer -----------------------------------------------------------------------------------------
114 (STEER_NOP
, STEER_CMD
, STEER_REQ
, STEER_REFRESH
) = range(4)
117 class _Steerer(Elaboratable
):
118 """Connects selected request to DFI interface
120 cas/ras/we/is_write/is_read are connected only when `cmd.valid & cmd.ready`.
121 Rank bits are decoded and used to drive cs_n in multi-rank systems,
122 STEER_REFRESH always enables all ranks.
126 commands : [Endpoint(cmd_request_rw_layout), ...]
127 Command streams to choose from. Must be of len=4 in the order:
128 NOP, CMD, REQ, REFRESH
129 NOP can be of type Record(cmd_request_rw_layout) instead, so that it is
130 always considered invalid (because of lack of the `valid` attribute).
132 DFI interface connected to PHY
136 sel : [Signal(range(len(commands))), ...], in
137 Signals for selecting which request gets connected to the corresponding
138 DFI phase. The signals should take one of the values from STEER_* to
142 def __init__(self
, commands
, dfi
):
143 if len(commands
) != 4:
144 raise ValueError("Commands is not the right size")
146 self
._commands
= commands
148 self
.sel
= [Signal(range(len(commands
))) for i
in range(len(dfi
.phases
))]
150 def elaborate(self
, platform
):
153 commands
= self
._commands
156 def valid_and(cmd
, attr
):
157 if not hasattr(cmd
, "valid"):
160 return cmd
.valid
& cmd
.ready
& getattr(cmd
, attr
)
162 for i
, (phase
, sel
) in enumerate(zip(dfi
.phases
, self
.sel
)):
163 nranks
= len(phase
.cs
)
164 rankbits
= log2_int(nranks
)
165 if hasattr(phase
, "reset"):
166 m
.d
.comb
+= phase
.reset
.eq(0)
167 m
.d
.comb
+= phase
.clk_en
.eq(Repl(Signal(reset
=1), nranks
))
168 if hasattr(phase
, "odt"):
169 # FIXME: add dynamic drive for multi-rank (will be needed for high frequencies)
170 m
.d
.comb
+= phase
.odt
.eq(Repl(Signal(reset
=1), nranks
))
172 rank_decoder
= Decoder(nranks
)
173 m
.submodules
+= rank_decoder
174 m
.d
.comb
+= rank_decoder
.i
.eq(
175 (Array(cmd
.ba
[-rankbits
:] for cmd
in commands
)[sel
]))
176 if i
== 0: # Select all ranks on refresh.
177 with m
.If(sel
== STEER_REFRESH
):
178 m
.d
.sync
+= phase
.cs
.eq(1)
180 m
.d
.sync
+= phase
.cs
.eq(rank_decoder
.o
)
182 m
.d
.sync
+= phase
.cs
.eq(rank_decoder
.o
)
183 m
.d
.sync
+= phase
.bank
.eq(Array(cmd
.ba
[:-rankbits
]
184 for cmd
in commands
)[sel
])
188 phase
.bank
.eq(Array(cmd
.ba
[:] for cmd
in commands
)[sel
]),
192 phase
.address
.eq(Array(cmd
.a
for cmd
in commands
)[sel
]),
193 phase
.cas
.eq(Array(valid_and(cmd
, "cas") for cmd
in commands
)[sel
]),
194 phase
.ras
.eq(Array(valid_and(cmd
, "ras") for cmd
in commands
)[sel
]),
195 phase
.we
.eq(Array(valid_and(cmd
, "we") for cmd
in commands
)[sel
])
198 rddata_ens
= Array(valid_and(cmd
, "is_read") for cmd
in commands
)
199 wrdata_ens
= Array(valid_and(cmd
, "is_write") for cmd
in commands
)
201 phase
.rddata_en
.eq(rddata_ens
[sel
]),
202 phase
.wrdata_en
.eq(wrdata_ens
[sel
])
207 class _AntiStarvation(Elaboratable
):
208 def __init__(self
, timeout
):
210 raise ValueError("Timeout values under 2 are not currently supported")
213 self
.max_time
= Signal(reset
=1)
214 self
._timeout
= timeout
216 def elaborate(self
, platform
):
219 if self
._timeout
> 0:
220 time
= Signal(range(self
._timeout
))
223 time
.eq(self
._timeout
-1),
226 with m
.Elif(time
!= 0):
227 m
.d
.sync
+= time
.eq(time
-1)
228 with m
.If(time
== 1):
229 m
.d
.sync
+= self
.max_time
.eq(1)
231 m
.d
.sync
+= self
.max_time
.eq(0)
233 m
.d
.comb
+= self
.max_time
.eq(0)
235 if platform
== "formal" and self
._timeout
> 0:
236 m
.d
.comb
+= Assert(self
.max_time
== (time
== 0))
240 class Multiplexer(Elaboratable
):
241 """Multplexes requets from BankMachines to DFI
243 This module multiplexes requests from BankMachines (and Refresher) and
244 connects them to DFI. Refresh commands are coordinated between the Refresher
245 and BankMachines to ensure there are no conflicts. Enforces required timings
246 between commands (some timings are enforced by BankMachines).
250 settings : ControllerSettings
251 Controller settings (with .phy, .geom and .timing settings)
252 bank_machines : [BankMachine, ...]
253 Bank machines that generate command requests to the Multiplexer
254 refresher : Refresher
255 Generates REFRESH command requests
257 DFI connected to the PHY
258 interface : LiteDRAMInterface
259 Data interface connected directly to LiteDRAMCrossbar
268 assert(settings
.phy
.nphases
== len(dfi
.phases
))
269 self
._settings
= settings
270 self
._bank
_machines
= bank_machines
271 self
._refresher
= refresher
273 self
._interface
= interface
275 def elaborate(self
, platform
):
278 settings
= self
._settings
279 bank_machines
= self
._bank
_machines
280 refresher
= self
._refresher
282 interface
= self
._interface
284 ras_allowed
= Signal(reset
=1)
285 cas_allowed
= Signal(reset
=1)
287 # Command choosing -------------------------------------------------------------------------
288 requests
= [bm
.cmd
for bm
in bank_machines
]
289 m
.submodules
.choose_cmd
= choose_cmd
= _CommandChooser(requests
)
290 m
.submodules
.choose_req
= choose_req
= _CommandChooser(requests
)
291 for i
, request
in enumerate(requests
):
292 m
.d
.comb
+= request
.ready
.eq(
293 choose_cmd
.ready
[i
] | choose_req
.ready
[i
])
294 if settings
.phy
.nphases
== 1:
295 # When only 1 phase, use choose_req for all requests
296 choose_cmd
= choose_req
297 m
.d
.comb
+= choose_req
.want_cmds
.eq(1)
298 m
.d
.comb
+= choose_req
.want_activates
.eq(ras_allowed
)
300 # Command steering -------------------------------------------------------------------------
301 nop
= Record(cmd_request_layout(settings
.geom
.addressbits
,
302 log2_int(len(bank_machines
))))
304 commands
= [nop
, choose_cmd
.cmd
, choose_req
.cmd
, refresher
.cmd
]
305 m
.submodules
.steerer
= steerer
= _Steerer(commands
, dfi
)
307 # tRRD timing (Row to Row delay) -----------------------------------------------------------
308 m
.submodules
.trrdcon
= trrdcon
= tXXDController(settings
.timing
.tRRD
)
309 m
.d
.comb
+= trrdcon
.valid
.eq(choose_cmd
.accept() & choose_cmd
.activate())
311 # tFAW timing (Four Activate Window) -------------------------------------------------------
312 m
.submodules
.tfawcon
= tfawcon
= tFAWController(settings
.timing
.tFAW
)
313 m
.d
.comb
+= tfawcon
.valid
.eq(choose_cmd
.accept() & choose_cmd
.activate())
315 # RAS control ------------------------------------------------------------------------------
316 m
.d
.comb
+= ras_allowed
.eq(trrdcon
.ready
& tfawcon
.ready
)
318 # tCCD timing (Column to Column delay) -----------------------------------------------------
319 m
.submodules
.tccdcon
= tccdcon
= tXXDController(settings
.timing
.tCCD
)
320 m
.d
.comb
+= tccdcon
.valid
.eq(choose_req
.accept() & (choose_req
.write() | choose_req
.read()))
322 # CAS control ------------------------------------------------------------------------------
323 m
.d
.comb
+= cas_allowed
.eq(tccdcon
.ready
)
325 # tWTR timing (Write to Read delay) --------------------------------------------------------
326 write_latency
= math
.ceil(settings
.phy
.cwl
/ settings
.phy
.nphases
)
327 m
.submodules
.twtrcon
= twtrcon
= tXXDController(
328 settings
.timing
.tWTR
+ write_latency
+
329 # tCCD must be added since tWTR begins after the transfer is complete
330 settings
.timing
.tCCD
if settings
.timing
.tCCD
is not None else 0)
331 m
.d
.comb
+= twtrcon
.valid
.eq(choose_req
.accept() & choose_req
.write())
333 # Read/write turnaround --------------------------------------------------------------------
334 reads
= Signal(len(requests
))
335 m
.d
.comb
+= reads
.eq(Cat([req
.valid
& req
.is_read
for req
in requests
]))
336 writes
= Signal(len(requests
))
337 m
.d
.comb
+= writes
.eq(Cat([req
.valid
& req
.is_write
for req
in requests
]))
339 # Anti Starvation --------------------------------------------------------------------------
340 m
.submodules
.read_antistarvation
= read_antistarvation
= _AntiStarvation(settings
.read_time
)
341 m
.submodules
.write_antistarvation
= write_antistarvation
= _AntiStarvation(settings
.write_time
)
343 # Refresh ----------------------------------------------------------------------------------
344 m
.d
.comb
+= [bm
.refresh_req
.eq(refresher
.cmd
.valid
) for bm
in bank_machines
]
345 bm_refresh_gnts
= Signal(len(bank_machines
))
346 m
.d
.comb
+= bm_refresh_gnts
.eq(Cat([bm
.refresh_gnt
for bm
in bank_machines
]))
348 # Datapath ---------------------------------------------------------------------------------
349 all_rddata
= [p
.rddata
for p
in dfi
.phases
]
350 all_wrdata
= [p
.wrdata
for p
in dfi
.phases
]
351 all_wrdata_mask
= [p
.wrdata_mask
for p
in dfi
.phases
]
353 interface
.rdata
.eq(Cat(*all_rddata
)),
354 Cat(*all_wrdata
).eq(interface
.wdata
),
355 Cat(*all_wrdata_mask
).eq(~interface
.wdata_we
)
358 # Control FSM ------------------------------------------------------------------------------
360 with m
.State("Read"):
362 read_antistarvation
.en
.eq(1),
363 choose_req
.want_reads
.eq(1),
366 for i
in range(settings
.phy
.nphases
):
367 if i
== settings
.phy
.rdphase
:
368 m
.d
.comb
+= steerer
.sel
[i
].eq(STEER_REQ
)
369 elif i
== settings
.phy
.rdcmdphase
:
370 m
.d
.comb
+= steerer
.sel
[i
].eq(STEER_CMD
)
372 with m
.If(settings
.phy
.nphases
== 1):
373 m
.d
.comb
+= choose_req
.cmd
.ready
.eq(cas_allowed
& (~choose_req
.activate() | ras_allowed
))
376 choose_cmd
.want_activates
.eq(ras_allowed
),
377 choose_cmd
.cmd
.ready
.eq(~choose_cmd
.activate() | ras_allowed
),
378 choose_req
.cmd
.ready
.eq(cas_allowed
),
381 with m
.If(writes
.any()):
382 # TODO: switch only after several cycles of ~reads.any()?
383 with m
.If(~reads
.any() | read_antistarvation
.max_time
):
386 with m
.If(bm_refresh_gnts
.all()):
389 with m
.State("Write"):
391 write_antistarvation
.en
.eq(1),
392 choose_req
.want_writes
.eq(1),
395 for i
in range(settings
.phy
.nphases
):
396 if i
== settings
.phy
.wrphase
:
397 m
.d
.comb
+= steerer
.sel
[i
].eq(STEER_REQ
)
398 elif i
== settings
.phy
.wrcmdphase
:
399 m
.d
.comb
+= steerer
.sel
[i
].eq(STEER_CMD
)
401 with m
.If(settings
.phy
.nphases
== 1):
402 m
.d
.comb
+= choose_req
.cmd
.ready
.eq(
403 cas_allowed
& (~choose_req
.activate() | ras_allowed
))
406 choose_cmd
.want_activates
.eq(ras_allowed
),
407 choose_cmd
.cmd
.ready
.eq(~choose_cmd
.activate() | ras_allowed
),
408 choose_req
.cmd
.ready
.eq(cas_allowed
),
411 with m
.If(reads
.any()):
412 with m
.If(~writes
.any() | write_antistarvation
.max_time
):
415 with m
.If(bm_refresh_gnts
.all()):
418 with m
.State("Refresh"):
420 steerer
.sel
[0].eq(STEER_REFRESH
),
421 refresher
.cmd
.ready
.eq(1),
423 with m
.If(refresher
.cmd
.last
):
427 with m
.If(twtrcon
.ready
):
430 # TODO: reduce this, actual limit is around (cl+1)/nphases
431 delayed_enter(m
, "RTW", "Write", settings
.phy
.read_latency
-1)