From 49fc83aa23045abee5d396ef5a9d96b80c03178d Mon Sep 17 00:00:00 2001 From: Tim Newsome Date: Fri, 29 Sep 2017 13:20:30 -0700 Subject: [PATCH] Fix tests to work in multi-gdb mode. The Gdb class now can handle connecting to more than one gdb. It enumerates the harts across all connections, and when asked to select a hart, it transparently sends future gdb commands to the correct instance. Multicore tests still have to be aware of some differences. The main one is that when executing 'c' in RTOS mode, all harts resume, while in multi-gdb mode only the current one resumes. Additionally, gdb doesn't set breakpoints until 'c' is issued, so the hart where breakpoints are set needs to be resumed before other harts might see them. --- debug/Makefile | 2 +- debug/gdbserver.py | 29 ++- debug/targets.py | 2 + .../targets/RISC-V/{spike.cfg => spike-1.cfg} | 0 debug/targets/RISC-V/spike-2.cfg | 19 ++ debug/targets/RISC-V/spike-rtos.cfg | 1 + debug/targets/RISC-V/spike32-2-rtos.py | 12 + debug/targets/RISC-V/spike32-2.py | 2 +- debug/targets/RISC-V/spike32.py | 2 +- debug/targets/RISC-V/spike64-2-rtos.py | 12 + debug/targets/RISC-V/spike64-2.py | 2 +- debug/targets/RISC-V/spike64.py | 2 +- debug/testlib.py | 238 ++++++++++++------ 13 files changed, 236 insertions(+), 87 deletions(-) rename debug/targets/RISC-V/{spike.cfg => spike-1.cfg} (100%) create mode 100644 debug/targets/RISC-V/spike-2.cfg create mode 100644 debug/targets/RISC-V/spike32-2-rtos.py create mode 100644 debug/targets/RISC-V/spike64-2-rtos.py diff --git a/debug/Makefile b/debug/Makefile index 33988dd..9f7cb2e 100644 --- a/debug/Makefile +++ b/debug/Makefile @@ -6,7 +6,7 @@ GDBSERVER_PY = $(src_dir)/gdbserver.py default: spike$(XLEN)-2 -all: pylint spike32 spike64 spike32-2 spike64-2 +all: pylint spike32 spike32-2 spike32-2-rtos spike64 spike64-2 spike64-2-rtos pylint: pylint --rcfile=pylint.rc `git ls-files '*.py'` diff --git a/debug/gdbserver.py b/debug/gdbserver.py index 135dab8..924f42a 100755 --- a/debug/gdbserver.py +++ b/debug/gdbserver.py @@ -363,7 +363,7 @@ class Hwbp2(DebugTest): self.exit() class TooManyHwbp(DebugTest): - def run(self): + def test(self): for i in range(30): self.gdb.hbreak("*rot13 + %d" % (i * 4)) @@ -476,21 +476,27 @@ class MulticoreRegTest(GdbTest): def test(self): # Run to main + # Hart 0 is the first to be resumed, so we have to set the breakpoint + # there. gdb won't actually set the breakpoint until we tell it to + # resume. + self.gdb.select_hart(self.target.harts[0]) self.gdb.b("main") - self.gdb.c() - for t in self.gdb.threads(): - assertIn("main", t.frame) + self.gdb.c_all() + for hart in self.target.harts: + self.gdb.select_hart(hart) + assertIn("main", self.gdb.where()) + self.gdb.select_hart(self.target.harts[0]) self.gdb.command("delete breakpoints") # Run through the entire loop. self.gdb.b("main_end") - self.gdb.c() + self.gdb.c_all() hart_ids = [] - for t in self.gdb.threads(): - assertIn("main_end", t.frame) + for hart in self.target.harts: + self.gdb.select_hart(hart) + assertIn("main_end", self.gdb.where()) # Check register values. - self.gdb.thread(t) hart_id = self.gdb.p("$x1") assertNotIn(hart_id, hart_ids) hart_ids.append(hart_id) @@ -505,12 +511,11 @@ class MulticoreRegTest(GdbTest): self.gdb.select_hart(hart) self.gdb.p("$x1=0x%x" % (hart.index * 0x800)) self.gdb.p("$pc=main_post_csrr") - self.gdb.c() - for t in self.gdb.threads(): - assertIn("main_end", t.frame) + self.gdb.c_all() for hart in self.target.harts: - # Check register values. self.gdb.select_hart(hart) + assertIn("main", self.gdb.where()) + # Check register values. for n in range(1, 32): value = self.gdb.p("$x%d" % n) assertEqual(value, hart.index * 0x800 + n - 1) diff --git a/debug/targets.py b/debug/targets.py index d09b576..624eb71 100644 --- a/debug/targets.py +++ b/debug/targets.py @@ -96,6 +96,8 @@ class Target(object): self.openocd_config_path) for i, hart in enumerate(self.harts): hart.index = i + if not hasattr(hart, 'id'): + hart.id = i if not hart.name: hart.name = "%s-%d" % (self.name, i) # Default link script to .lds diff --git a/debug/targets/RISC-V/spike.cfg b/debug/targets/RISC-V/spike-1.cfg similarity index 100% rename from debug/targets/RISC-V/spike.cfg rename to debug/targets/RISC-V/spike-1.cfg diff --git a/debug/targets/RISC-V/spike-2.cfg b/debug/targets/RISC-V/spike-2.cfg new file mode 100644 index 0000000..17526ec --- /dev/null +++ b/debug/targets/RISC-V/spike-2.cfg @@ -0,0 +1,19 @@ +# Connect to a mult-icore RISC-V target, exposing each hart as a thread. +adapter_khz 10000 + +interface remote_bitbang +remote_bitbang_host $::env(REMOTE_BITBANG_HOST) +remote_bitbang_port $::env(REMOTE_BITBANG_PORT) + +set _CHIPNAME riscv +jtag newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x10e31913 + +set _TARGETNAME_0 $_CHIPNAME.cpu0 +set _TARGETNAME_1 $_CHIPNAME.cpu1 +target create $_TARGETNAME_0 riscv -chain-position $_CHIPNAME.cpu -coreid 0 +target create $_TARGETNAME_1 riscv -chain-position $_CHIPNAME.cpu -coreid 1 + +gdb_report_data_abort enable + +init +reset halt diff --git a/debug/targets/RISC-V/spike-rtos.cfg b/debug/targets/RISC-V/spike-rtos.cfg index 9b1841c..799e3cb 100644 --- a/debug/targets/RISC-V/spike-rtos.cfg +++ b/debug/targets/RISC-V/spike-rtos.cfg @@ -1,3 +1,4 @@ +# Connect to a mult-icore RISC-V target, exposing each hart as a thread. adapter_khz 10000 interface remote_bitbang diff --git a/debug/targets/RISC-V/spike32-2-rtos.py b/debug/targets/RISC-V/spike32-2-rtos.py new file mode 100644 index 0000000..a7b9a1c --- /dev/null +++ b/debug/targets/RISC-V/spike32-2-rtos.py @@ -0,0 +1,12 @@ +import targets +import testlib + +import spike32 # pylint: disable=import-error + +class spike32_2(targets.Target): + harts = [spike32.spike32_hart(), spike32.spike32_hart()] + openocd_config_path = "spike-rtos.cfg" + timeout_sec = 30 + + def create(self): + return testlib.Spike(self) diff --git a/debug/targets/RISC-V/spike32-2.py b/debug/targets/RISC-V/spike32-2.py index a7b9a1c..719009d 100644 --- a/debug/targets/RISC-V/spike32-2.py +++ b/debug/targets/RISC-V/spike32-2.py @@ -5,7 +5,7 @@ import spike32 # pylint: disable=import-error class spike32_2(targets.Target): harts = [spike32.spike32_hart(), spike32.spike32_hart()] - openocd_config_path = "spike-rtos.cfg" + openocd_config_path = "spike-2.cfg" timeout_sec = 30 def create(self): diff --git a/debug/targets/RISC-V/spike32.py b/debug/targets/RISC-V/spike32.py index bcb5892..809463c 100644 --- a/debug/targets/RISC-V/spike32.py +++ b/debug/targets/RISC-V/spike32.py @@ -11,7 +11,7 @@ class spike32_hart(targets.Hart): class spike32(targets.Target): harts = [spike32_hart()] - openocd_config_path = "spike.cfg" + openocd_config_path = "spike-1.cfg" timeout_sec = 30 def create(self): diff --git a/debug/targets/RISC-V/spike64-2-rtos.py b/debug/targets/RISC-V/spike64-2-rtos.py new file mode 100644 index 0000000..d65d2ab --- /dev/null +++ b/debug/targets/RISC-V/spike64-2-rtos.py @@ -0,0 +1,12 @@ +import targets +import testlib + +import spike64 # pylint: disable=import-error + +class spike64_2_rtos(targets.Target): + harts = [spike64.spike64_hart(), spike64.spike64_hart()] + openocd_config_path = "spike-rtos.cfg" + timeout_sec = 30 + + def create(self): + return testlib.Spike(self) diff --git a/debug/targets/RISC-V/spike64-2.py b/debug/targets/RISC-V/spike64-2.py index 4f6f1ff..709ebbe 100644 --- a/debug/targets/RISC-V/spike64-2.py +++ b/debug/targets/RISC-V/spike64-2.py @@ -5,7 +5,7 @@ import spike64 # pylint: disable=import-error class spike64_2(targets.Target): harts = [spike64.spike64_hart(), spike64.spike64_hart()] - openocd_config_path = "spike-rtos.cfg" + openocd_config_path = "spike-2.cfg" timeout_sec = 30 def create(self): diff --git a/debug/targets/RISC-V/spike64.py b/debug/targets/RISC-V/spike64.py index 9c37f87..2cd67a5 100644 --- a/debug/targets/RISC-V/spike64.py +++ b/debug/targets/RISC-V/spike64.py @@ -11,7 +11,7 @@ class spike64_hart(targets.Hart): class spike64(targets.Target): harts = [spike64_hart()] - openocd_config_path = "spike.cfg" + openocd_config_path = "spike-1.cfg" timeout_sec = 30 def create(self): diff --git a/debug/testlib.py b/debug/testlib.py index 0bdb433..385034b 100644 --- a/debug/testlib.py +++ b/debug/testlib.py @@ -1,4 +1,5 @@ import collections +import os import os.path import random import re @@ -51,8 +52,6 @@ def compile(args, xlen=32): # pylint: disable=redefined-builtin raise Exception("Compile failed!") class Spike(object): - logname = "spike-%d.log" % os.getpid() - def __init__(self, target, halted=False, timeout=None, with_jtag_gdb=True): """Launch spike. Return tuple of its process and the port it's running on.""" @@ -68,11 +67,13 @@ class Spike(object): "programs/checksum.c", "programs/tiny-malloc.c", "programs/infinite_loop.S", "-DDEFINE_MALLOC", "-DDEFINE_FREE") cmd.append(self.infinite_loop) - logfile = open(self.logname, "w") - logfile.write("+ %s\n" % " ".join(cmd)) - logfile.flush() + self.logfile = tempfile.NamedTemporaryFile(prefix="spike-", + suffix=".log") + self.logname = self.logfile.name + self.logfile.write("+ %s\n" % " ".join(cmd)) + self.logfile.flush() self.process = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=logfile, stderr=logfile) + stdout=self.logfile, stderr=self.logfile) if with_jtag_gdb: self.port = None @@ -223,8 +224,7 @@ class Openocd(object): logfile.write("+ %s\n" % " ".join(cmd)) logfile.flush() - self.ports = [] - self.port = None + self.gdb_ports = [] self.process = self.start(cmd, logfile) def start(self, cmd, logfile): @@ -238,31 +238,32 @@ class Openocd(object): # attempt too early. start = time.time() messaged = False + fd = open(Openocd.logname, "r") while True: - log = open(Openocd.logname).read() + line = fd.readline() + if not line: + if not process.poll() is None: + raise Exception("OpenOCD exited early.") + time.sleep(0.1) + continue + m = re.search(r"Listening on port (\d+) for gdb connections", - log) + line) if m: - if not self.ports: - self.port = int(m.group(1)) - self.ports.append(int(m.group(1))) + self.gdb_ports.append(int(m.group(1))) - if "telnet server disabled" in log: - break + if "telnet server disabled" in line: + return process - if not process.poll() is None: - raise Exception( - "OpenOCD exited before completing riscv_examine()") if not messaged and time.time() - start > 1: messaged = True print "Waiting for OpenOCD to start..." if (time.time() - start) > self.timeout: - raise Exception("ERROR: Timed out waiting for OpenOCD to " + raise Exception("Timed out waiting for OpenOCD to " "listen for gdb") - return process + except Exception: - header("OpenOCD log") - sys.stdout.write(log) + print_log(Openocd.logname) raise def __del__(self): @@ -303,42 +304,109 @@ class CannotAccess(Exception): Exception.__init__(self) self.address = address -Thread = collections.namedtuple('Thread', ('id', 'target_id', 'name', - 'frame')) +Thread = collections.namedtuple('Thread', ('id', 'description', 'target_id', + 'name', 'frame')) class Gdb(object): - logfile = tempfile.NamedTemporaryFile(prefix="gdb", suffix=".log") - logname = logfile.name - print "GDB Temporary Log File: %s" % logname - - def __init__(self, - cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb")): - self.child = pexpect.spawn(cmd) - self.child.logfile = open(self.logname, "w") - self.child.logfile.write("+ %s\n" % cmd) - self.wait() - self.command("set confirm off") - self.command("set width 0") - self.command("set height 0") - # Force consistency. - self.command("set print entry-values no") + """A single gdb class which can interact with one or more gdb instances.""" + + # pylint: disable=too-many-public-methods + + def __init__(self, ports, + cmd=os.path.expandvars("$RISCV/bin/riscv64-unknown-elf-gdb"), + binary=None): + assert ports + + self.stack = [] + + self.logfiles = [] + self.children = [] + for port in ports: + logfile = tempfile.NamedTemporaryFile(prefix="gdb@%d-" % port, + suffix=".log") + self.logfiles.append(logfile) + child = pexpect.spawn(cmd) + child.logfile = logfile + child.logfile.write("+ %s\n" % cmd) + self.children.append(child) + self.active_child = self.children[0] + + self.harts = {} + for port, child in zip(ports, self.children): + self.select_child(child) + self.wait() + self.command("set confirm off") + self.command("set width 0") + self.command("set height 0") + # Force consistency. + self.command("set print entry-values no") + self.command("target extended-remote localhost:%d" % port) + if binary: + self.command("file %s" % binary) + threads = self.threads() + for t in threads: + hartid = None + if t.name: + m = re.search(r"Hart (\d+)", t.name) + if m: + hartid = int(m.group(1)) + if hartid is None: + if self.harts: + hartid = max(self.harts) + 1 + else: + hartid = 0 + self.harts[hartid] = (child, t) + + def __del__(self): + for child in self.children: + del child + + def lognames(self): + return [logfile.name for logfile in self.logfiles] + + def select_child(self, child): + self.active_child = child def select_hart(self, hart): - output = self.command("thread %d" % (hart.index + 1)) + child, thread = self.harts[hart.id] + self.select_child(child) + output = self.command("thread %s" % thread.id) assert "Unknown" not in output + def push_state(self): + self.stack.append({ + 'active_child': self.active_child + }) + + def pop_state(self): + state = self.stack.pop() + self.active_child = state['active_child'] + def wait(self): """Wait for prompt.""" - self.child.expect(r"\(gdb\)") + self.active_child.expect(r"\(gdb\)") def command(self, command, timeout=6000): """timeout is in seconds""" - self.child.sendline(command) - self.child.expect("\n", timeout=timeout) - self.child.expect(r"\(gdb\)", timeout=timeout) - return self.child.before.strip() + self.active_child.sendline(command) + self.active_child.expect("\n", timeout=timeout) + self.active_child.expect(r"\(gdb\)", timeout=timeout) + return self.active_child.before.strip() + + def global_command(self, command): + """Execute this command on every gdb that we control.""" + with PrivateState(self): + for child in self.children: + self.select_child(child) + self.command(command) def c(self, wait=True, timeout=-1, async=False): + """ + Dumb c command. + In RTOS mode, gdb will resume all harts. + In multi-gdb mode, this command will just go to the current gdb, so + will only resume one hart. + """ if async: async = "&" else: @@ -348,13 +416,24 @@ class Gdb(object): assert "Continuing" in output return output else: - self.child.sendline("c%s" % async) - self.child.expect("Continuing") + self.active_child.sendline("c%s" % async) + self.active_child.expect("Continuing") + + def c_all(self): + """Resume every hart.""" + with PrivateState(self): + for child in self.children: + child.sendline("c") + child.expect("Continuing") + + # Now wait for them all to halt + for child in self.children: + child.expect(r"\(gdb\)") def interrupt(self): - self.child.send("\003") - self.child.expect(r"\(gdb\)", timeout=6000) - return self.child.before.strip() + self.active_child.send("\003") + self.active_child.expect(r"\(gdb\)", timeout=6000) + return self.active_child.before.strip() def x(self, address, size='w'): output = self.command("x/%s %s" % (size, address)) @@ -417,17 +496,32 @@ class Gdb(object): threads = [] for line in output.splitlines(): m = re.match( - r"[\s\*]*(\d+)\s*Thread (\d+)\s*\(Name: ([^\)]+)\s*(.*)", - line) + r"[\s\*]*(\d+)\s*" + r"(Remote target|Thread (\d+)\s*\(Name: ([^\)]+))" + r"\s*(.*)", line) if m: threads.append(Thread(*m.groups())) - if not threads: - threads.append(Thread('1', '1', 'Default', '???')) + assert threads + #>>>if not threads: + #>>> threads.append(Thread('1', '1', 'Default', '???')) return threads def thread(self, thread): return self.command("thread %s" % thread.id) + def where(self): + return self.command("where 1") + +class PrivateState(object): + def __init__(self, gdb): + self.gdb = gdb + + def __enter__(self): + self.gdb.push_state() + + def __exit__(self, _type, _value, _traceback): + self.gdb.pop_state() + def run_all_tests(module, target, parsed): if not os.path.exists(parsed.logs): os.makedirs(parsed.logs) @@ -469,7 +563,7 @@ def run_tests(parsed, target, todo): log_name = os.path.join(parsed.logs, "%s-%s-%s.log" % (time.strftime("%Y%m%d-%H%M%S"), type(target).__name__, name)) log_fd = open(log_name, 'w') - print "Running %s > %s ..." % (name, log_name), + print "[%s] Starting > %s" % (name, log_name) instance = definition(target, hart) sys.stdout.flush() log_fd.write("Test: %s\n" % name) @@ -478,12 +572,12 @@ def run_tests(parsed, target, todo): real_stdout = sys.stdout sys.stdout = log_fd try: - result = instance.run() + result = instance.run(real_stdout) log_fd.write("Result: %s\n" % result) finally: sys.stdout = real_stdout log_fd.write("Time elapsed: %.2fs\n" % (time.time() - start)) - print "%s in %.2fs" % (result, time.time() - start) + print "[%s] %s in %.2fs" % (name, result, time.time() - start) if result not in good_results and parsed.print_failures: sys.stdout.write(open(log_name).read()) sys.stdout.flush() @@ -589,7 +683,7 @@ class BaseTest(object): def postMortem(self): pass - def run(self): + def run(self, real_stdout): """ If compile_args is set, compile a program and set self.binary. @@ -608,6 +702,8 @@ class BaseTest(object): try: self.classSetup() + real_stdout.write("[%s] Temporary logs: %s\n" % ( + type(self).__name__, ", ".join(self.logs))) self.setup() result = self.test() # pylint: disable=no-member except TestNotApplicable: @@ -622,7 +718,12 @@ class BaseTest(object): print e.message header("Traceback") traceback.print_exc(file=sys.stdout) - self.postMortem() + try: + self.postMortem() + except Exception as e: # pylint: disable=broad-except + header("postMortem Exception") + print e + traceback.print_exc(file=sys.stdout) return result finally: @@ -645,25 +746,22 @@ class GdbTest(BaseTest): BaseTest.classSetup(self) if gdb_cmd: - self.gdb = Gdb(gdb_cmd) + self.gdb = Gdb(self.server.gdb_ports, gdb_cmd, binary=self.binary) else: - self.gdb = Gdb() + self.gdb = Gdb(self.server.gdb_ports, binary=self.binary) - self.logs.append(self.gdb.logname) + self.logs += self.gdb.lognames() - if self.binary: - self.gdb.command("file %s" % self.binary) if self.target: - self.gdb.command("set arch riscv:rv%d" % self.hart.xlen) - self.gdb.command("set remotetimeout %d" % self.target.timeout_sec) - if self.server.port: - self.gdb.command( - "target extended-remote localhost:%d" % self.server.port) - self.gdb.select_hart(self.hart) + self.gdb.global_command("set arch riscv:rv%d" % self.hart.xlen) + self.gdb.global_command("set remotetimeout %d" % + self.target.timeout_sec) for cmd in self.target.gdb_setup: self.gdb.command(cmd) + self.gdb.select_hart(self.hart) + # FIXME: OpenOCD doesn't handle PRIV now #self.gdb.p("$priv=3") -- 2.30.2