test/run_readelf_tests.py

   1 #!/usr/bin/env python
   2 #-------------------------------------------------------------------------------
   3 # test/run_readelf_tests.py
   4 #
   5 # Automatic test runner for elftools & readelf
   6 #
   7 # Eli Bendersky (eliben@gmail.com)
   8 # This code is in the public domain
   9 #-------------------------------------------------------------------------------
  10 import argparse
  11 from difflib import SequenceMatcher
  12 import logging
  13 from multiprocessing import Pool
  14 import os
  15 import platform
  16 import re
  17 import sys
  18 import time
  19
  20 from utils import run_exe, is_in_rootdir, dump_output_to_temp_files
  21
  22 # Make it possible to run this file from the root dir of pyelftools without
  23 # installing pyelftools; useful for CI testing, etc.
  24 sys.path[0:0] = ['.']
  25
  26 # Create a global logger object
  27 testlog = logging.getLogger('run_tests')
  28 testlog.setLevel(logging.DEBUG)
  29 testlog.addHandler(logging.StreamHandler(sys.stdout))
  30
  31 # Set the path for calling readelf. We carry our own version of readelf around,
  32 # because binutils tend to change its output even between daily builds of the
  33 # same minor release and keeping track is a headache.
  34 if platform.system() == "Darwin": # MacOS
  35     READELF_PATH = 'greadelf'
  36 else:
  37     READELF_PATH = 'test/external_tools/readelf'
  38     if not os.path.exists(READELF_PATH):
  39         READELF_PATH = 'readelf'
  40
  41
  42 def discover_testfiles(rootdir):
  43     """ Discover test files in the given directory. Yield them one by one.
  44     """
  45     for filename in os.listdir(rootdir):
  46         _, ext = os.path.splitext(filename)
  47         if ext == '.elf':
  48             yield os.path.join(rootdir, filename)
  49
  50
  51 def run_test_on_file(filename, verbose=False, opt=None):
  52     """ Runs a test on the given input filename. Return True if all test
  53         runs succeeded.
  54         If opt is specified, rather that going over the whole
  55         set of supported readelf options, the test will only
  56         run for one option.
  57     """
  58     success = True
  59     testlog.info("Test file '%s'" % filename)
  60     if opt is None:
  61         options = [
  62             '-e', '-d', '-s', '-n', '-r', '-x.text', '-p.shstrtab', '-V',
  63             '--debug-dump=info', '--debug-dump=decodedline',
  64             '--debug-dump=frames', '--debug-dump=frames-interp',
  65             '--debug-dump=aranges', '--debug-dump=pubtypes',
  66             '--debug-dump=pubnames', '--debug-dump=loc'
  67             ]
  68     else:
  69         options = [opt]
  70
  71     for option in options:
  72         if verbose: testlog.info("..option='%s'" % option)
  73
  74         # TODO(zlobober): this is a dirty hack to make tests work for ELF core
  75         # dump notes. Making it work properly requires a pretty deep
  76         # investigation of how original readelf formats the output.
  77         if "core" in filename and option == "-n":
  78             if verbose:
  79                 testlog.warning("....will fail because corresponding part of readelf.py is not implemented yet")
  80                 testlog.info('.......................SKIPPED')
  81             continue
  82
  83         # sevaa says: there is another shorted out test; in dwarf_lineprogramv5.elf, the two bytes at 0x2072 were
  84         # patched from 0x07 0x10 to 00 00.
  85         # Those represented the second instruction in the first FDE in .eh_frame. This changed the instruction
  86         # from "DW_CFA_undefined 16" to two NOPs.
  87         # GNU readelf had a bug here, had to work around. See PR #411.
  88
  89         # stdouts will be a 2-element list: output of readelf and output
  90         # of scripts/readelf.py
  91         stdouts = []
  92         for exe_path in [READELF_PATH, 'scripts/readelf.py']:
  93             args = [option, filename]
  94             if verbose: testlog.info("....executing: '%s %s'" % (
  95                 exe_path, ' '.join(args)))
  96             t1 = time.time()
  97             rc, stdout = run_exe(exe_path, args)
  98             if verbose: testlog.info("....elapsed: %s" % (time.time() - t1,))
  99             if rc != 0:
 100                 testlog.error("@@ aborting - '%s %s' returned '%s'" % (exe_path, option, rc))
 101                 return False
 102             stdouts.append(stdout)
 103         if verbose: testlog.info('....comparing output...')
 104         t1 = time.time()
 105         rc, errmsg = compare_output(*stdouts)
 106         if verbose: testlog.info("....elapsed: %s" % (time.time() - t1,))
 107         if rc:
 108             if verbose: testlog.info('.......................SUCCESS')
 109         else:
 110             success = False
 111             testlog.info('.......................FAIL')
 112             testlog.info('....for option "%s"' % option)
 113             testlog.info('....Output #1 is readelf, Output #2 is pyelftools')
 114             testlog.info('@@ ' + errmsg)
 115             dump_output_to_temp_files(testlog, *stdouts)
 116     return success
 117
 118
 119 def compare_output(s1, s2):
 120     """ Compare stdout strings s1 and s2.
 121         s1 is from readelf, s2 from elftools readelf.py
 122         Return pair success, errmsg. If comparison succeeds, success is True
 123         and errmsg is empty. Otherwise success is False and errmsg holds a
 124         description of the mismatch.
 125
 126         Note: this function contains some rather horrible hacks to ignore
 127         differences which are not important for the verification of pyelftools.
 128         This is due to some intricacies of binutils's readelf which pyelftools
 129         doesn't currently implement, features that binutils doesn't support,
 130         or silly inconsistencies in the output of readelf, which I was reluctant
 131         to replicate. Read the documentation for more details.
 132     """
 133     def prepare_lines(s):
 134         return [line for line in s.lower().splitlines() if line.strip() != '']
 135
 136     lines1 = prepare_lines(s1)
 137     lines2 = prepare_lines(s2)
 138
 139     flag_in_debug_line_section = False
 140
 141     if len(lines1) != len(lines2):
 142         return False, 'Number of lines different: %s vs %s' % (
 143                 len(lines1), len(lines2))
 144
 145     for i in range(len(lines1)):
 146         if lines1[i].endswith('debug_line section:'):
 147             # .debug_line or .zdebug_line
 148             flag_in_debug_line_section = True
 149
 150         # readelf spelling error for GNU property notes
 151         lines1[i] = lines1[i].replace('procesor-specific type', 'processor-specific type')
 152
 153         # Compare ignoring whitespace
 154         lines1_parts = lines1[i].split()
 155         lines2_parts = lines2[i].split()
 156
 157         if ''.join(lines1_parts) != ''.join(lines2_parts):
 158             ok = False
 159
 160             try:
 161                 # Ignore difference in precision of hex representation in the
 162                 # last part (i.e. 008f3b vs 8f3b)
 163                 if (''.join(lines1_parts[:-1]) == ''.join(lines2_parts[:-1]) and
 164                     int(lines1_parts[-1], 16) == int(lines2_parts[-1], 16)):
 165                     ok = True
 166             except ValueError:
 167                 pass
 168
 169             sm = SequenceMatcher()
 170             sm.set_seqs(lines1[i], lines2[i])
 171             changes = sm.get_opcodes()
 172             if flag_in_debug_line_section:
 173                 # readelf outputs an additional "View" column: ignore it
 174                 if len(lines1_parts) >= 2 and lines1_parts[-2] == 'view':
 175                     ok = True
 176                 else:
 177                     # Fast check special-cased for the only ELF we have which
 178                     # has this information (dwarf_gnuops4.so.elf)
 179                     ok = (    lines1_parts[-2:] == ['1', 'x']
 180                           and lines2_parts[-1] == 'x')
 181             elif '[...]' in lines1[i]:
 182                 # Special case truncations with ellipsis like these:
 183                 #     .note.gnu.bu[...]        redelf
 184                 #     .note.gnu.build-i        pyelftools
 185                 # Or more complex for symbols with versions, like these:
 186                 #     _unw[...]@gcc_3.0        readelf
 187                 #     _unwind_resume@gcc_3.0   pyelftools
 188                 for p1, p2 in zip(lines1_parts, lines2_parts):
 189                     dots_start = p1.find('[...]')
 190                     if dots_start != -1:
 191                         break
 192                 ok = p1.endswith('[...]') and p1[:dots_start] == p2[:dots_start]
 193                 if not ok:
 194                     dots_end = dots_start + 5
 195                     if len(p1) > dots_end and p1[dots_end] == '@':
 196                         ok = (    p1[:dots_start] == p2[:dots_start]
 197                               and p1[p1.rfind('@'):] == p2[p2.rfind('@'):])
 198             elif 'at_const_value' in lines1[i]:
 199                 # On 32-bit machines, readelf doesn't correctly represent
 200                 # some boundary LEB128 numbers
 201                 val = lines2_parts[-1]
 202                 num2 = int(val, 16 if val.startswith('0x') else 10)
 203                 if num2 <= -2**31 and '32' in platform.architecture()[0]:
 204                     ok = True
 205             elif 'os/abi' in lines1[i]:
 206                 if 'unix - gnu' in lines1[i] and 'unix - linux' in lines2[i]:
 207                     ok = True
 208             elif len(lines1_parts) == 3 and lines1_parts[2] == 'nt_gnu_property_type_0':
 209                 # readelf does not seem to print a readable description for this
 210                 ok = lines1_parts == lines2_parts[:3]
 211             else:
 212                 for s in ('t (tls)', 'l (large)', 'd (mbind)'):
 213                     if s in lines1[i] or s in lines2[i]:
 214                         ok = True
 215                         break
 216             if not ok:
 217                 errmsg = 'Mismatch on line #%s:\n>>%s<<\n>>%s<<\n (%r)' % (
 218                     i, lines1[i], lines2[i], changes)
 219                 return False, errmsg
 220     return True, ''
 221
 222
 223 def main():
 224     if not is_in_rootdir():
 225         testlog.error('Error: Please run me from the root dir of pyelftools!')
 226         return 1
 227
 228     argparser = argparse.ArgumentParser(
 229         usage='usage: %(prog)s [options] [file] [file] ...',
 230         prog='run_readelf_tests.py')
 231     argparser.add_argument('files', nargs='*', help='files to run tests on')
 232     argparser.add_argument(
 233         '--parallel', action='store_true',
 234         help='run tests in parallel; always runs all tests w/o verbose')
 235     argparser.add_argument('-V', '--verbose',
 236                            action='store_true', dest='verbose',
 237                            help='verbose output')
 238     argparser.add_argument(
 239         '-k', '--keep-going',
 240         action='store_true', dest='keep_going',
 241         help="Run all tests, don't stop at the first failure")
 242     argparser.add_argument('--opt',
 243         action='store', dest='opt', metavar='<readelf-option>',
 244         help= 'Limit the test one one readelf option.')
 245     args = argparser.parse_args()
 246
 247     if args.parallel:
 248         if args.verbose or args.keep_going == False:
 249             print('WARNING: parallel mode disables verbosity and always keeps going')
 250
 251     if args.verbose:
 252         testlog.info('Running in verbose mode')
 253         testlog.info('Python executable = %s' % sys.executable)
 254         testlog.info('readelf path = %s' % READELF_PATH)
 255         testlog.info('Given list of files: %s' % args.files)
 256
 257     # If file names are given as command-line arguments, only these files
 258     # are taken as inputs. Otherwise, autodiscovery is performed.
 259     if len(args.files) > 0:
 260         filenames = args.files
 261     else:
 262         filenames = sorted(discover_testfiles('test/testfiles_for_readelf'))
 263
 264     if len(filenames) > 1 and args.parallel:
 265         pool = Pool()
 266         results = pool.map(run_test_on_file, filenames)
 267         failures = results.count(False)
 268     else:
 269         failures = 0
 270         for filename in filenames:
 271             if not run_test_on_file(filename, args.verbose, args.opt):
 272                 failures += 1
 273                 if not args.keep_going:
 274                     break
 275
 276     if failures == 0:
 277         testlog.info('\nConclusion: SUCCESS')
 278         return 0
 279     elif args.keep_going:
 280         testlog.info('\nConclusion: FAIL ({}/{})'.format(
 281             failures, len(filenames)))
 282         return 1
 283     else:
 284         testlog.info('\nConclusion: FAIL')
 285         return 1
 286
 287
 288 if __name__ == '__main__':
 289     sys.exit(main())