test/run_readelf_tests.py

   1 #!/usr/bin/env python
   2 #-------------------------------------------------------------------------------
   3 # test/run_readelf_tests.py
   4 #
   5 # Automatic test runner for elftools & readelf
   6 #
   7 # Eli Bendersky (eliben@gmail.com)
   8 # This code is in the public domain
   9 #-------------------------------------------------------------------------------
  10 import argparse
  11 from difflib import SequenceMatcher
  12 import logging
  13 from multiprocessing import Pool
  14 import os
  15 import platform
  16 import re
  17 import sys
  18 import time
  19
  20 from utils import run_exe, is_in_rootdir, dump_output_to_temp_files
  21
  22 # Make it possible to run this file from the root dir of pyelftools without
  23 # installing pyelftools; useful for CI testing, etc.
  24 sys.path[0:0] = ['.']
  25
  26 # Create a global logger object
  27 testlog = logging.getLogger('run_tests')
  28 testlog.setLevel(logging.DEBUG)
  29 testlog.addHandler(logging.StreamHandler(sys.stdout))
  30
  31 # Set the path for calling readelf. We carry our own version of readelf around,
  32 # because binutils tend to change its output even between daily builds of the
  33 # same minor release and keeping track is a headache.
  34 if platform.system() == "Darwin": # MacOS
  35     READELF_PATH = 'greadelf'
  36 else:
  37     READELF_PATH = 'test/external_tools/readelf'
  38     if not os.path.exists(READELF_PATH):
  39         READELF_PATH = 'readelf'
  40
  41
  42 def discover_testfiles(rootdir):
  43     """ Discover test files in the given directory. Yield them one by one.
  44     """
  45     for filename in os.listdir(rootdir):
  46         _, ext = os.path.splitext(filename)
  47         if ext == '.elf':
  48             yield os.path.join(rootdir, filename)
  49
  50
  51 def run_test_on_file(filename, verbose=False, opt=None):
  52     """ Runs a test on the given input filename. Return True if all test
  53         runs succeeded.
  54         If opt is specified, rather that going over the whole
  55         set of supported readelf options, the test will only
  56         run for one option.
  57     """
  58     success = True
  59     testlog.info("Test file '%s'" % filename)
  60     if opt is None:
  61         options = [
  62             '-e', '-d', '-s', '-n', '-r', '-x.text', '-p.shstrtab', '-V',
  63             '--debug-dump=info', '--debug-dump=decodedline',
  64             '--debug-dump=frames', '--debug-dump=frames-interp',
  65             '--debug-dump=aranges', '--debug-dump=pubtypes',
  66             '--debug-dump=pubnames', '--debug-dump=loc'
  67             ]
  68     else:
  69         options = [opt]
  70
  71     for option in options:
  72         if verbose: testlog.info("..option='%s'" % option)
  73
  74         # TODO(zlobober): this is a dirty hack to make tests work for ELF core
  75         # dump notes. Making it work properly requires a pretty deep
  76         # investigation of how original readelf formats the output.
  77         if "core" in filename and option == "-n":
  78             if verbose:
  79                 testlog.warning("....will fail because corresponding part of readelf.py is not implemented yet")
  80                 testlog.info('.......................SKIPPED')
  81             continue
  82
  83         # stdouts will be a 2-element list: output of readelf and output
  84         # of scripts/readelf.py
  85         stdouts = []
  86         for exe_path in [READELF_PATH, 'scripts/readelf.py']:
  87             args = [option, filename]
  88             if verbose: testlog.info("....executing: '%s %s'" % (
  89                 exe_path, ' '.join(args)))
  90             t1 = time.time()
  91             rc, stdout = run_exe(exe_path, args)
  92             if verbose: testlog.info("....elapsed: %s" % (time.time() - t1,))
  93             if rc != 0:
  94                 testlog.error("@@ aborting - '%s %s' returned '%s'" % (exe_path, option, rc))
  95                 return False
  96             stdouts.append(stdout)
  97         if verbose: testlog.info('....comparing output...')
  98         t1 = time.time()
  99         rc, errmsg = compare_output(*stdouts)
 100         if verbose: testlog.info("....elapsed: %s" % (time.time() - t1,))
 101         if rc:
 102             if verbose: testlog.info('.......................SUCCESS')
 103         else:
 104             success = False
 105             testlog.info('.......................FAIL')
 106             testlog.info('....for option "%s"' % option)
 107             testlog.info('....Output #1 is readelf, Output #2 is pyelftools')
 108             testlog.info('@@ ' + errmsg)
 109             dump_output_to_temp_files(testlog, *stdouts)
 110     return success
 111
 112
 113 def compare_output(s1, s2):
 114     """ Compare stdout strings s1 and s2.
 115         s1 is from readelf, s2 from elftools readelf.py
 116         Return pair success, errmsg. If comparison succeeds, success is True
 117         and errmsg is empty. Otherwise success is False and errmsg holds a
 118         description of the mismatch.
 119
 120         Note: this function contains some rather horrible hacks to ignore
 121         differences which are not important for the verification of pyelftools.
 122         This is due to some intricacies of binutils's readelf which pyelftools
 123         doesn't currently implement, features that binutils doesn't support,
 124         or silly inconsistencies in the output of readelf, which I was reluctant
 125         to replicate. Read the documentation for more details.
 126     """
 127     def prepare_lines(s):
 128         return [line for line in s.lower().splitlines() if line.strip() != '']
 129
 130     lines1 = prepare_lines(s1)
 131     lines2 = prepare_lines(s2)
 132
 133     flag_after_symtable = False
 134
 135     if len(lines1) != len(lines2):
 136         return False, 'Number of lines different: %s vs %s' % (
 137                 len(lines1), len(lines2))
 138
 139     for i in range(len(lines1)):
 140         if 'symbol table' in lines1[i]:
 141             flag_after_symtable = True
 142
 143         # Compare ignoring whitespace
 144         lines1_parts = lines1[i].split()
 145         lines2_parts = lines2[i].split()
 146
 147         if ''.join(lines1_parts) != ''.join(lines2_parts):
 148             ok = False
 149
 150             try:
 151                 # Ignore difference in precision of hex representation in the
 152                 # last part (i.e. 008f3b vs 8f3b)
 153                 if (''.join(lines1_parts[:-1]) == ''.join(lines2_parts[:-1]) and
 154                     int(lines1_parts[-1], 16) == int(lines2_parts[-1], 16)):
 155                     ok = True
 156             except ValueError:
 157                 pass
 158
 159             sm = SequenceMatcher()
 160             sm.set_seqs(lines1[i], lines2[i])
 161             changes = sm.get_opcodes()
 162             if flag_after_symtable:
 163                 # Detect readelf's adding @ with lib and version after
 164                 # symbol name.
 165                 if (    len(changes) == 2 and changes[1][0] == 'delete' and
 166                         lines1[i][changes[1][1]] == '@'):
 167                     ok = True
 168             elif 'at_const_value' in lines1[i]:
 169                 # On 32-bit machines, readelf doesn't correctly represent
 170                 # some boundary LEB128 numbers
 171                 val = lines2_parts[-1]
 172                 num2 = int(val, 16 if val.startswith('0x') else 10)
 173                 if num2 <= -2**31 and '32' in platform.architecture()[0]:
 174                     ok = True
 175             elif 'os/abi' in lines1[i]:
 176                 if 'unix - gnu' in lines1[i] and 'unix - linux' in lines2[i]:
 177                     ok = True
 178             elif (  'unknown at value' in lines1[i] and
 179                     'dw_at_apple' in lines2[i]):
 180                 ok = True
 181             else:
 182                 for s in ('t (tls)', 'l (large)'):
 183                     if s in lines1[i] or s in lines2[i]:
 184                         ok = True
 185                         break
 186             if not ok:
 187                 errmsg = 'Mismatch on line #%s:\n>>%s<<\n>>%s<<\n (%r)' % (
 188                     i, lines1[i], lines2[i], changes)
 189                 return False, errmsg
 190     return True, ''
 191
 192
 193 def main():
 194     if not is_in_rootdir():
 195         testlog.error('Error: Please run me from the root dir of pyelftools!')
 196         return 1
 197
 198     argparser = argparse.ArgumentParser(
 199         usage='usage: %(prog)s [options] [file] [file] ...',
 200         prog='run_readelf_tests.py')
 201     argparser.add_argument('files', nargs='*', help='files to run tests on')
 202     argparser.add_argument(
 203         '--parallel', action='store_true',
 204         help='run tests in parallel; always runs all tests w/o verbose')
 205     argparser.add_argument('-V', '--verbose',
 206                            action='store_true', dest='verbose',
 207                            help='verbose output')
 208     argparser.add_argument(
 209         '-k', '--keep-going',
 210         action='store_true', dest='keep_going',
 211         help="Run all tests, don't stop at the first failure")
 212     argparser.add_argument('--opt',
 213         action='store', dest='opt', metavar='<readelf-option>',
 214         help= 'Limit the test one one readelf option.')
 215     args = argparser.parse_args()
 216
 217     if args.parallel:
 218         if args.verbose or args.keep_going == False:
 219             print('WARNING: parallel mode disables verbosity and always keeps going')
 220
 221     if args.verbose:
 222         testlog.info('Running in verbose mode')
 223         testlog.info('Python executable = %s' % sys.executable)
 224         testlog.info('readelf path = %s' % READELF_PATH)
 225         testlog.info('Given list of files: %s' % args.files)
 226
 227     # If file names are given as command-line arguments, only these files
 228     # are taken as inputs. Otherwise, autodiscovery is performed.
 229     if len(args.files) > 0:
 230         filenames = args.files
 231     else:
 232         filenames = sorted(discover_testfiles('test/testfiles_for_readelf'))
 233
 234     if len(filenames) > 1 and args.parallel:
 235         pool = Pool()
 236         results = pool.map(run_test_on_file, filenames)
 237         failures = results.count(False)
 238     else:
 239         failures = 0
 240         for filename in filenames:
 241             if not run_test_on_file(filename, args.verbose, args.opt):
 242                 failures += 1
 243                 if not args.keep_going:
 244                     break
 245
 246     if failures == 0:
 247         testlog.info('\nConclusion: SUCCESS')
 248         return 0
 249     elif args.keep_going:
 250         testlog.info('\nConclusion: FAIL ({}/{})'.format(
 251             failures, len(filenames)))
 252         return 1
 253     else:
 254         testlog.info('\nConclusion: FAIL')
 255         return 1
 256
 257
 258 if __name__ == '__main__':
 259     sys.exit(main())