test/run_readelf_tests.py

   1 #!/usr/bin/env python
   2 #-------------------------------------------------------------------------------
   3 # test/run_readelf_tests.py
   4 #
   5 # Automatic test runner for elftools & readelf
   6 #
   7 # Eli Bendersky (eliben@gmail.com)
   8 # This code is in the public domain
   9 #-------------------------------------------------------------------------------
  10 import argparse
  11 from difflib import SequenceMatcher
  12 import logging
  13 from multiprocessing import Pool
  14 import os
  15 import platform
  16 import re
  17 import sys
  18 import time
  19
  20 from utils import run_exe, is_in_rootdir, dump_output_to_temp_files
  21
  22 # Make it possible to run this file from the root dir of pyelftools without
  23 # installing pyelftools; useful for CI testing, etc.
  24 sys.path[0:0] = ['.']
  25
  26 # Create a global logger object
  27 testlog = logging.getLogger('run_tests')
  28 testlog.setLevel(logging.DEBUG)
  29 testlog.addHandler(logging.StreamHandler(sys.stdout))
  30
  31 # Set the path for calling readelf. We carry our own version of readelf around,
  32 # because binutils tend to change its output even between daily builds of the
  33 # same minor release and keeping track is a headache.
  34 if platform.system() == "Darwin": # MacOS
  35     READELF_PATH = 'greadelf'
  36 elif platform.system() == "Windows":
  37     # Point the environment variable READELF at Cygwin's readelf.exe, or some other Windows build
  38     READELF_PATH = os.environ.get('READELF', "readelf.exe")
  39 else:
  40     READELF_PATH = 'test/external_tools/readelf'
  41     if not os.path.exists(READELF_PATH):
  42         READELF_PATH = 'readelf'
  43
  44
  45 def discover_testfiles(rootdir):
  46     """ Discover test files in the given directory. Yield them one by one.
  47     """
  48     for filename in os.listdir(rootdir):
  49         _, ext = os.path.splitext(filename)
  50         if ext == '.elf':
  51             yield os.path.join(rootdir, filename)
  52
  53
  54 def run_test_on_file(filename, verbose=False, opt=None):
  55     """ Runs a test on the given input filename. Return True if all test
  56         runs succeeded.
  57         If opt is specified, rather that going over the whole
  58         set of supported readelf options, the test will only
  59         run for one option.
  60     """
  61     success = True
  62     testlog.info("Test file '%s'" % filename)
  63     if opt is None:
  64         options = [
  65             '-e', '-d', '-s', '-n', '-r', '-x.text', '-p.shstrtab', '-V',
  66             '--debug-dump=info', '--debug-dump=decodedline',
  67             '--debug-dump=frames', '--debug-dump=frames-interp',
  68             '--debug-dump=aranges', '--debug-dump=pubtypes',
  69             '--debug-dump=pubnames', '--debug-dump=loc',
  70             '--debug-dump=Ranges'
  71             ]
  72     else:
  73         options = [opt]
  74
  75     for option in options:
  76         if verbose: testlog.info("..option='%s'" % option)
  77
  78         # TODO(zlobober): this is a dirty hack to make tests work for ELF core
  79         # dump notes. Making it work properly requires a pretty deep
  80         # investigation of how original readelf formats the output.
  81         if "core" in filename and option == "-n":
  82             if verbose:
  83                 testlog.warning("....will fail because corresponding part of readelf.py is not implemented yet")
  84                 testlog.info('.......................SKIPPED')
  85             continue
  86
  87         # sevaa says: there is another shorted out test; in dwarf_lineprogramv5.elf, the two bytes at 0x2072 were
  88         # patched from 0x07 0x10 to 00 00.
  89         # Those represented the second instruction in the first FDE in .eh_frame. This changed the instruction
  90         # from "DW_CFA_undefined 16" to two NOPs.
  91         # GNU readelf 2.38 had a bug here, had to work around:
  92         # https://sourceware.org/bugzilla/show_bug.cgi?id=29250
  93         # It's been fixed in the binutils' master since, but the latest master will break a lot.
  94         # Same patch in  dwarf_test_versions_mix.elf at 0x2061: 07 10 -> 00 00
  95
  96         # stdouts will be a 2-element list: output of readelf and output
  97         # of scripts/readelf.py
  98         stdouts = []
  99         for exe_path in [READELF_PATH, 'scripts/readelf.py']:
 100             args = [option, filename]
 101             if verbose: testlog.info("....executing: '%s %s'" % (
 102                 exe_path, ' '.join(args)))
 103             t1 = time.time()
 104             rc, stdout = run_exe(exe_path, args)
 105             if verbose: testlog.info("....elapsed: %s" % (time.time() - t1,))
 106             if rc != 0:
 107                 testlog.error("@@ aborting - '%s %s' returned '%s'" % (exe_path, option, rc))
 108                 return False
 109             stdouts.append(stdout)
 110         if verbose: testlog.info('....comparing output...')
 111         t1 = time.time()
 112         rc, errmsg = compare_output(*stdouts)
 113         if verbose: testlog.info("....elapsed: %s" % (time.time() - t1,))
 114         if rc:
 115             if verbose: testlog.info('.......................SUCCESS')
 116         else:
 117             success = False
 118             testlog.info('.......................FAIL')
 119             testlog.info('....for file %s' % filename)
 120             testlog.info('....for option "%s"' % option)
 121             testlog.info('....Output #1 is readelf, Output #2 is pyelftools')
 122             testlog.info('@@ ' + errmsg)
 123             dump_output_to_temp_files(testlog, filename, option, *stdouts)
 124     return success
 125
 126
 127 def compare_output(s1, s2):
 128     """ Compare stdout strings s1 and s2.
 129         s1 is from readelf, s2 from elftools readelf.py
 130         Return pair success, errmsg. If comparison succeeds, success is True
 131         and errmsg is empty. Otherwise success is False and errmsg holds a
 132         description of the mismatch.
 133
 134         Note: this function contains some rather horrible hacks to ignore
 135         differences which are not important for the verification of pyelftools.
 136         This is due to some intricacies of binutils's readelf which pyelftools
 137         doesn't currently implement, features that binutils doesn't support,
 138         or silly inconsistencies in the output of readelf, which I was reluctant
 139         to replicate. Read the documentation for more details.
 140     """
 141     def prepare_lines(s):
 142         return [line for line in s.lower().splitlines() if line.strip() != '']
 143
 144     lines1 = prepare_lines(s1)
 145     lines2 = prepare_lines(s2)
 146
 147     flag_in_debug_line_section = False
 148
 149     if len(lines1) != len(lines2):
 150         return False, 'Number of lines different: %s vs %s' % (
 151                 len(lines1), len(lines2))
 152
 153     # Position of the View column in the output file, if parsing readelf..decodedline
 154     # output, and the GNU readelf output contains the View column. Otherwise stays -1.
 155     view_col_position = -1
 156     for i in range(len(lines1)):
 157         if lines1[i].endswith('debug_line section:'):
 158             # .debug_line or .zdebug_line
 159             flag_in_debug_line_section = True
 160
 161         # readelf spelling error for GNU property notes
 162         lines1[i] = lines1[i].replace('procesor-specific type', 'processor-specific type')
 163
 164         # The view column position may change from CU to CU:
 165         if view_col_position >= 0 and lines1[i].startswith('cu:'):
 166             view_col_position = -1
 167
 168         # Check if readelf..decodedline output line contains the view column
 169         if flag_in_debug_line_section and lines1[i].startswith('file name') and view_col_position < 0:
 170             view_col_position = lines1[i].find("view")
 171             stmt_col_position = lines1[i].find("stmt")
 172
 173         # Excise the View column from the table, if any.
 174         # View_col_position is only set to a nonzero number if one of the previous
 175         # lines was a table header line with a "view" in it.
 176         # We assume careful formatting on GNU readelf's part - View column values
 177         # are not out of line with the View header.
 178         if view_col_position >= 0 and not lines1[i].endswith(':'):
 179             lines1[i] = lines1[i][:view_col_position] + lines1[i][stmt_col_position:]
 180
 181         # Compare ignoring whitespace
 182         lines1_parts = lines1[i].split()
 183         lines2_parts = lines2[i].split()
 184
 185         if ''.join(lines1_parts) != ''.join(lines2_parts):
 186             ok = False
 187
 188             try:
 189                 # Ignore difference in precision of hex representation in the
 190                 # last part (i.e. 008f3b vs 8f3b)
 191                 if (''.join(lines1_parts[:-1]) == ''.join(lines2_parts[:-1]) and
 192                     int(lines1_parts[-1], 16) == int(lines2_parts[-1], 16)):
 193                     ok = True
 194             except ValueError:
 195                 pass
 196
 197             sm = SequenceMatcher()
 198             sm.set_seqs(lines1[i], lines2[i])
 199             changes = sm.get_opcodes()
 200             if '[...]' in lines1[i]:
 201                 # Special case truncations with ellipsis like these:
 202                 #     .note.gnu.bu[...]        redelf
 203                 #     .note.gnu.build-i        pyelftools
 204                 # Or more complex for symbols with versions, like these:
 205                 #     _unw[...]@gcc_3.0        readelf
 206                 #     _unwind_resume@gcc_3.0   pyelftools
 207                 for p1, p2 in zip(lines1_parts, lines2_parts):
 208                     dots_start = p1.find('[...]')
 209                     if dots_start != -1:
 210                         break
 211                 ok = p1.endswith('[...]') and p1[:dots_start] == p2[:dots_start]
 212                 if not ok:
 213                     dots_end = dots_start + 5
 214                     if len(p1) > dots_end and p1[dots_end] == '@':
 215                         ok = (    p1[:dots_start] == p2[:dots_start]
 216                               and p1[p1.rfind('@'):] == p2[p2.rfind('@'):])
 217             elif 'at_const_value' in lines1[i]:
 218                 # On 32-bit machines, readelf doesn't correctly represent
 219                 # some boundary LEB128 numbers
 220                 val = lines2_parts[-1]
 221                 num2 = int(val, 16 if val.startswith('0x') else 10)
 222                 if num2 <= -2**31 and '32' in platform.architecture()[0]:
 223                     ok = True
 224             elif 'os/abi' in lines1[i]:
 225                 if 'unix - gnu' in lines1[i] and 'unix - linux' in lines2[i]:
 226                     ok = True
 227             elif len(lines1_parts) == 3 and lines1_parts[2] == 'nt_gnu_property_type_0':
 228                 # readelf does not seem to print a readable description for this
 229                 ok = lines1_parts == lines2_parts[:3]
 230             else:
 231                 for s in ('t (tls)', 'l (large)', 'd (mbind)'):
 232                     if s in lines1[i] or s in lines2[i]:
 233                         ok = True
 234                         break
 235             if not ok:
 236                 errmsg = 'Mismatch on line #%s:\n>>%s<<\n>>%s<<\n (%r)' % (
 237                     i, lines1[i], lines2[i], changes)
 238                 return False, errmsg
 239     return True, ''
 240
 241
 242 def main():
 243     if not is_in_rootdir():
 244         testlog.error('Error: Please run me from the root dir of pyelftools!')
 245         return 1
 246
 247     argparser = argparse.ArgumentParser(
 248         usage='usage: %(prog)s [options] [file] [file] ...',
 249         prog='run_readelf_tests.py')
 250     argparser.add_argument('files', nargs='*', help='files to run tests on')
 251     argparser.add_argument(
 252         '--parallel', action='store_true',
 253         help='run tests in parallel; always runs all tests w/o verbose')
 254     argparser.add_argument('-V', '--verbose',
 255                            action='store_true', dest='verbose',
 256                            help='verbose output')
 257     argparser.add_argument(
 258         '-k', '--keep-going',
 259         action='store_true', dest='keep_going',
 260         help="Run all tests, don't stop at the first failure")
 261     argparser.add_argument('--opt',
 262         action='store', dest='opt', metavar='<readelf-option>',
 263         help= 'Limit the test one one readelf option.')
 264     args = argparser.parse_args()
 265
 266     if args.parallel:
 267         if args.verbose or args.keep_going == False:
 268             print('WARNING: parallel mode disables verbosity and always keeps going')
 269
 270     if args.verbose:
 271         testlog.info('Running in verbose mode')
 272         testlog.info('Python executable = %s' % sys.executable)
 273         testlog.info('readelf path = %s' % READELF_PATH)
 274         testlog.info('Given list of files: %s' % args.files)
 275
 276     # If file names are given as command-line arguments, only these files
 277     # are taken as inputs. Otherwise, autodiscovery is performed.
 278     if len(args.files) > 0:
 279         filenames = args.files
 280     else:
 281         filenames = sorted(discover_testfiles('test/testfiles_for_readelf'))
 282
 283     if len(filenames) > 1 and args.parallel:
 284         pool = Pool()
 285         results = pool.map(run_test_on_file, filenames)
 286         failures = results.count(False)
 287     else:
 288         failures = 0
 289         for filename in filenames:
 290             if not run_test_on_file(filename, args.verbose, args.opt):
 291                 failures += 1
 292                 if not args.keep_going:
 293                     break
 294
 295     if failures == 0:
 296         testlog.info('\nConclusion: SUCCESS')
 297         return 0
 298     elif args.keep_going:
 299         testlog.info('\nConclusion: FAIL ({}/{})'.format(
 300             failures, len(filenames)))
 301         return 1
 302     else:
 303         testlog.info('\nConclusion: FAIL')
 304         return 1
 305
 306
 307 if __name__ == '__main__':
 308     sys.exit(main())