scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2022 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 try:
  60     subprocess.run
  61 except:
  62     class _CompletedProcess:
  63         def __init__(self, args, returncode, stdout=None, stderr=None):
  64             self.args = args
  65             self.returncode = returncode
  66             self.stdout = stdout
  67             self.stderr = stderr
  68
  69     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  70         assert(timeout is None)
  71         with subprocess.Popen(*popenargs, **kwargs) as process:
  72             try:
  73                 stdout, stderr = process.communicate(input)
  74             except:
  75                 process.kill()
  76                 process.wait()
  77                 raise
  78             returncode = process.poll()
  79             if check and returncode:
  80                 raise subprocess.CalledProcessError(returncode, popenargs)
  81         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  82
  83     subprocess.run = _run
  84
  85
  86 class Context(object):
  87     """The global state associated with builds in a given directory."""
  88
  89     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  90                  full_gcc, action, shallow=False):
  91         """Initialize the context."""
  92         self.topdir = topdir
  93         self.parallelism = parallelism
  94         self.keep = keep
  95         self.replace_sources = replace_sources
  96         self.strip = strip
  97         self.full_gcc = full_gcc
  98         self.shallow = shallow
  99         self.srcdir = os.path.join(topdir, 'src')
 100         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 101         self.build_state_json = os.path.join(topdir, 'build-state.json')
 102         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 103         self.installdir = os.path.join(topdir, 'install')
 104         self.host_libraries_installdir = os.path.join(self.installdir,
 105                                                       'host-libraries')
 106         self.builddir = os.path.join(topdir, 'build')
 107         self.logsdir = os.path.join(topdir, 'logs')
 108         self.logsdir_old = os.path.join(topdir, 'logs-old')
 109         self.makefile = os.path.join(self.builddir, 'Makefile')
 110         self.wrapper = os.path.join(self.builddir, 'wrapper')
 111         self.save_logs = os.path.join(self.builddir, 'save-logs')
 112         self.script_text = self.get_script_text()
 113         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 114             self.build_triplet = self.get_build_triplet()
 115             self.glibc_version = self.get_glibc_version()
 116         self.configs = {}
 117         self.glibc_configs = {}
 118         self.makefile_pieces = ['.PHONY: all\n']
 119         self.add_all_configs()
 120         self.load_versions_json()
 121         self.load_build_state_json()
 122         self.status_log_list = []
 123         self.email_warning = False
 124
 125     def get_script_text(self):
 126         """Return the text of this script."""
 127         with open(sys.argv[0], 'r') as f:
 128             return f.read()
 129
 130     def exec_self(self):
 131         """Re-execute this script with the same arguments."""
 132         sys.stdout.flush()
 133         os.execv(sys.executable, [sys.executable] + sys.argv)
 134
 135     def get_build_triplet(self):
 136         """Determine the build triplet with config.guess."""
 137         config_guess = os.path.join(self.component_srcdir('gcc'),
 138                                     'config.guess')
 139         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 140                                 check=True, universal_newlines=True).stdout
 141         return cg_out.rstrip()
 142
 143     def get_glibc_version(self):
 144         """Determine the glibc version number (major.minor)."""
 145         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 146         with open(version_h, 'r') as f:
 147             lines = f.readlines()
 148         starttext = '#define VERSION "'
 149         for l in lines:
 150             if l.startswith(starttext):
 151                 l = l[len(starttext):]
 152                 l = l.rstrip('"\n')
 153                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 154                 return '%s.%s' % m.group(1, 2)
 155         print('error: could not determine glibc version')
 156         exit(1)
 157
 158     def add_all_configs(self):
 159         """Add all known glibc build configurations."""
 160         self.add_config(arch='aarch64',
 161                         os_name='linux-gnu',
 162                         extra_glibcs=[{'variant': 'disable-multi-arch',
 163                                        'cfg': ['--disable-multi-arch']}])
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='arc',
 167                         os_name='linux-gnu',
 168                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 169         self.add_config(arch='arc',
 170                         os_name='linux-gnuhf',
 171                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 172         self.add_config(arch='arceb',
 173                         os_name='linux-gnu',
 174                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 175         self.add_config(arch='alpha',
 176                         os_name='linux-gnu')
 177         self.add_config(arch='arm',
 178                         os_name='linux-gnueabi',
 179                         extra_glibcs=[{'variant': 'v4t',
 180                                        'ccopts': '-march=armv4t'}])
 181         self.add_config(arch='armeb',
 182                         os_name='linux-gnueabi')
 183         self.add_config(arch='armeb',
 184                         os_name='linux-gnueabi',
 185                         variant='be8',
 186                         gcc_cfg=['--with-arch=armv7-a'])
 187         self.add_config(arch='arm',
 188                         os_name='linux-gnueabihf',
 189                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 190                         extra_glibcs=[{'variant': 'v7a',
 191                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 192                                       {'variant': 'thumb',
 193                                        'ccopts':
 194                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 195                                       {'variant': 'v7a-disable-multi-arch',
 196                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 197                                        'cfg': ['--disable-multi-arch']}])
 198         self.add_config(arch='armeb',
 199                         os_name='linux-gnueabihf',
 200                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabihf',
 203                         variant='be8',
 204                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 205                                  '--with-fpu=vfpv3'])
 206         self.add_config(arch='csky',
 207                         os_name='linux-gnuabiv2',
 208                         variant='soft',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='csky',
 211                         os_name='linux-gnuabiv2',
 212                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 213         self.add_config(arch='hppa',
 214                         os_name='linux-gnu')
 215         self.add_config(arch='i686',
 216                         os_name='gnu')
 217         self.add_config(arch='ia64',
 218                         os_name='linux-gnu',
 219                         first_gcc_cfg=['--with-system-libunwind'],
 220                         binutils_cfg=['--enable-obsolete'])
 221         self.add_config(arch='loongarch64',
 222                         os_name='linux-gnu',
 223                         variant='lp64d',
 224                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 225         self.add_config(arch='m68k',
 226                         os_name='linux-gnu',
 227                         gcc_cfg=['--disable-multilib'])
 228         self.add_config(arch='m68k',
 229                         os_name='linux-gnu',
 230                         variant='coldfire',
 231                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 232         self.add_config(arch='m68k',
 233                         os_name='linux-gnu',
 234                         variant='coldfire-soft',
 235                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 236                                  '--disable-multilib'])
 237         self.add_config(arch='microblaze',
 238                         os_name='linux-gnu',
 239                         gcc_cfg=['--disable-multilib'])
 240         self.add_config(arch='microblazeel',
 241                         os_name='linux-gnu',
 242                         gcc_cfg=['--disable-multilib'])
 243         self.add_config(arch='mips64',
 244                         os_name='linux-gnu',
 245                         gcc_cfg=['--with-mips-plt'],
 246                         glibcs=[{'variant': 'n32'},
 247                                 {'arch': 'mips',
 248                                  'ccopts': '-mabi=32'},
 249                                 {'variant': 'n64',
 250                                  'ccopts': '-mabi=64'}])
 251         self.add_config(arch='mips64',
 252                         os_name='linux-gnu',
 253                         variant='soft',
 254                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 255                         glibcs=[{'variant': 'n32-soft'},
 256                                 {'variant': 'soft',
 257                                  'arch': 'mips',
 258                                  'ccopts': '-mabi=32'},
 259                                 {'variant': 'n64-soft',
 260                                  'ccopts': '-mabi=64'}])
 261         self.add_config(arch='mips64',
 262                         os_name='linux-gnu',
 263                         variant='nan2008',
 264                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 265                                  '--with-arch-64=mips64r2',
 266                                  '--with-arch-32=mips32r2'],
 267                         glibcs=[{'variant': 'n32-nan2008'},
 268                                 {'variant': 'nan2008',
 269                                  'arch': 'mips',
 270                                  'ccopts': '-mabi=32'},
 271                                 {'variant': 'n64-nan2008',
 272                                  'ccopts': '-mabi=64'}])
 273         self.add_config(arch='mips64',
 274                         os_name='linux-gnu',
 275                         variant='nan2008-soft',
 276                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 277                                  '--with-arch-64=mips64r2',
 278                                  '--with-arch-32=mips32r2',
 279                                  '--with-float=soft'],
 280                         glibcs=[{'variant': 'n32-nan2008-soft'},
 281                                 {'variant': 'nan2008-soft',
 282                                  'arch': 'mips',
 283                                  'ccopts': '-mabi=32'},
 284                                 {'variant': 'n64-nan2008-soft',
 285                                  'ccopts': '-mabi=64'}])
 286         self.add_config(arch='mips64el',
 287                         os_name='linux-gnu',
 288                         gcc_cfg=['--with-mips-plt'],
 289                         glibcs=[{'variant': 'n32'},
 290                                 {'arch': 'mipsel',
 291                                  'ccopts': '-mabi=32'},
 292                                 {'variant': 'n64',
 293                                  'ccopts': '-mabi=64'}])
 294         self.add_config(arch='mips64el',
 295                         os_name='linux-gnu',
 296                         variant='soft',
 297                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 298                         glibcs=[{'variant': 'n32-soft'},
 299                                 {'variant': 'soft',
 300                                  'arch': 'mipsel',
 301                                  'ccopts': '-mabi=32'},
 302                                 {'variant': 'n64-soft',
 303                                  'ccopts': '-mabi=64'}])
 304         self.add_config(arch='mips64el',
 305                         os_name='linux-gnu',
 306                         variant='nan2008',
 307                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 308                                  '--with-arch-64=mips64r2',
 309                                  '--with-arch-32=mips32r2'],
 310                         glibcs=[{'variant': 'n32-nan2008'},
 311                                 {'variant': 'nan2008',
 312                                  'arch': 'mipsel',
 313                                  'ccopts': '-mabi=32'},
 314                                 {'variant': 'n64-nan2008',
 315                                  'ccopts': '-mabi=64'}])
 316         self.add_config(arch='mips64el',
 317                         os_name='linux-gnu',
 318                         variant='nan2008-soft',
 319                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 320                                  '--with-arch-64=mips64r2',
 321                                  '--with-arch-32=mips32r2',
 322                                  '--with-float=soft'],
 323                         glibcs=[{'variant': 'n32-nan2008-soft'},
 324                                 {'variant': 'nan2008-soft',
 325                                  'arch': 'mipsel',
 326                                  'ccopts': '-mabi=32'},
 327                                 {'variant': 'n64-nan2008-soft',
 328                                  'ccopts': '-mabi=64'}])
 329         self.add_config(arch='mipsisa64r6el',
 330                         os_name='linux-gnu',
 331                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 332                                  '--with-arch-64=mips64r6',
 333                                  '--with-arch-32=mips32r6',
 334                                  '--with-float=hard'],
 335                         glibcs=[{'variant': 'n32'},
 336                                 {'arch': 'mipsisa32r6el',
 337                                  'ccopts': '-mabi=32'},
 338                                 {'variant': 'n64',
 339                                  'ccopts': '-mabi=64'}])
 340         self.add_config(arch='nios2',
 341                         os_name='linux-gnu')
 342         self.add_config(arch='or1k',
 343                         os_name='linux-gnu',
 344                         variant='soft',
 345                         gcc_cfg=['--with-multilib-list=mcmov'])
 346         self.add_config(arch='powerpc',
 347                         os_name='linux-gnu',
 348                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 349                         extra_glibcs=[{'variant': 'power4',
 350                                        'ccopts': '-mcpu=power4',
 351                                        'cfg': ['--with-cpu=power4']}])
 352         self.add_config(arch='powerpc',
 353                         os_name='linux-gnu',
 354                         variant='soft',
 355                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 356                                  '--enable-secureplt'])
 357         self.add_config(arch='powerpc64',
 358                         os_name='linux-gnu',
 359                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 360         self.add_config(arch='powerpc64le',
 361                         os_name='linux-gnu',
 362                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 363                         extra_glibcs=[{'variant': 'disable-multi-arch',
 364                                        'cfg': ['--disable-multi-arch']}])
 365         self.add_config(arch='riscv32',
 366                         os_name='linux-gnu',
 367                         variant='rv32imac-ilp32',
 368                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 369                                  '--disable-multilib'])
 370         self.add_config(arch='riscv32',
 371                         os_name='linux-gnu',
 372                         variant='rv32imafdc-ilp32',
 373                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 374                                  '--disable-multilib'])
 375         self.add_config(arch='riscv32',
 376                         os_name='linux-gnu',
 377                         variant='rv32imafdc-ilp32d',
 378                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 379                                  '--disable-multilib'])
 380         self.add_config(arch='riscv64',
 381                         os_name='linux-gnu',
 382                         variant='rv64imac-lp64',
 383                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 384                                  '--disable-multilib'])
 385         self.add_config(arch='riscv64',
 386                         os_name='linux-gnu',
 387                         variant='rv64imafdc-lp64',
 388                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 389                                  '--disable-multilib'])
 390         self.add_config(arch='riscv64',
 391                         os_name='linux-gnu',
 392                         variant='rv64imafdc-lp64d',
 393                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 394                                  '--disable-multilib'])
 395         self.add_config(arch='s390x',
 396                         os_name='linux-gnu',
 397                         glibcs=[{},
 398                                 {'arch': 's390', 'ccopts': '-m31'}],
 399                         extra_glibcs=[{'variant': 'O3',
 400                                        'cflags': '-O3'}])
 401         self.add_config(arch='sh3',
 402                         os_name='linux-gnu')
 403         self.add_config(arch='sh3eb',
 404                         os_name='linux-gnu')
 405         self.add_config(arch='sh4',
 406                         os_name='linux-gnu')
 407         self.add_config(arch='sh4eb',
 408                         os_name='linux-gnu')
 409         self.add_config(arch='sh4',
 410                         os_name='linux-gnu',
 411                         variant='soft',
 412                         gcc_cfg=['--without-fp'])
 413         self.add_config(arch='sh4eb',
 414                         os_name='linux-gnu',
 415                         variant='soft',
 416                         gcc_cfg=['--without-fp'])
 417         self.add_config(arch='sparc64',
 418                         os_name='linux-gnu',
 419                         glibcs=[{},
 420                                 {'arch': 'sparcv9',
 421                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 422                         extra_glibcs=[{'variant': 'leon3',
 423                                        'arch' : 'sparcv8',
 424                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 425                                       {'variant': 'disable-multi-arch',
 426                                        'cfg': ['--disable-multi-arch']},
 427                                       {'variant': 'disable-multi-arch',
 428                                        'arch': 'sparcv9',
 429                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 430                                        'cfg': ['--disable-multi-arch']}])
 431         self.add_config(arch='x86_64',
 432                         os_name='linux-gnu',
 433                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 434                         glibcs=[{},
 435                                 {'variant': 'x32', 'ccopts': '-mx32'},
 436                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 437                         extra_glibcs=[{'variant': 'disable-multi-arch',
 438                                        'cfg': ['--disable-multi-arch']},
 439                                       {'variant': 'minimal',
 440                                        'cfg': ['--disable-multi-arch',
 441                                                '--disable-profile',
 442                                                '--disable-timezone-tools',
 443                                                '--disable-mathvec',
 444                                                '--disable-tunables',
 445                                                '--disable-crypt',
 446                                                '--disable-experimental-malloc',
 447                                                '--disable-build-nscd',
 448                                                '--disable-nscd']},
 449                                       {'variant': 'no-pie',
 450                                        'cfg': ['--disable-default-pie']},
 451                                       {'variant': 'x32-no-pie',
 452                                        'ccopts': '-mx32',
 453                                        'cfg': ['--disable-default-pie']},
 454                                       {'variant': 'no-pie',
 455                                        'arch': 'i686',
 456                                        'ccopts': '-m32 -march=i686',
 457                                        'cfg': ['--disable-default-pie']},
 458                                       {'variant': 'disable-multi-arch',
 459                                        'arch': 'i686',
 460                                        'ccopts': '-m32 -march=i686',
 461                                        'cfg': ['--disable-multi-arch']},
 462                                       {'arch': 'i486',
 463                                        'ccopts': '-m32 -march=i486'},
 464                                       {'arch': 'i586',
 465                                        'ccopts': '-m32 -march=i586'}])
 466
 467     def add_config(self, **args):
 468         """Add an individual build configuration."""
 469         cfg = Config(self, **args)
 470         if cfg.name in self.configs:
 471             print('error: duplicate config %s' % cfg.name)
 472             exit(1)
 473         self.configs[cfg.name] = cfg
 474         for c in cfg.all_glibcs:
 475             if c.name in self.glibc_configs:
 476                 print('error: duplicate glibc config %s' % c.name)
 477                 exit(1)
 478             self.glibc_configs[c.name] = c
 479
 480     def component_srcdir(self, component):
 481         """Return the source directory for a given component, e.g. gcc."""
 482         return os.path.join(self.srcdir, component)
 483
 484     def component_builddir(self, action, config, component, subconfig=None):
 485         """Return the directory to use for a build."""
 486         if config is None:
 487             # Host libraries.
 488             assert subconfig is None
 489             return os.path.join(self.builddir, action, component)
 490         if subconfig is None:
 491             return os.path.join(self.builddir, action, config, component)
 492         else:
 493             # glibc build as part of compiler build.
 494             return os.path.join(self.builddir, action, config, component,
 495                                 subconfig)
 496
 497     def compiler_installdir(self, config):
 498         """Return the directory in which to install a compiler."""
 499         return os.path.join(self.installdir, 'compilers', config)
 500
 501     def compiler_bindir(self, config):
 502         """Return the directory in which to find compiler binaries."""
 503         return os.path.join(self.compiler_installdir(config), 'bin')
 504
 505     def compiler_sysroot(self, config):
 506         """Return the sysroot directory for a compiler."""
 507         return os.path.join(self.compiler_installdir(config), 'sysroot')
 508
 509     def glibc_installdir(self, config):
 510         """Return the directory in which to install glibc."""
 511         return os.path.join(self.installdir, 'glibcs', config)
 512
 513     def run_builds(self, action, configs):
 514         """Run the requested builds."""
 515         if action == 'checkout':
 516             self.checkout(configs)
 517             return
 518         if action == 'bot-cycle':
 519             if configs:
 520                 print('error: configurations specified for bot-cycle')
 521                 exit(1)
 522             self.bot_cycle()
 523             return
 524         if action == 'bot':
 525             if configs:
 526                 print('error: configurations specified for bot')
 527                 exit(1)
 528             self.bot()
 529             return
 530         if action in ('host-libraries', 'list-compilers',
 531                       'list-glibcs') and configs:
 532             print('error: configurations specified for ' + action)
 533             exit(1)
 534         if action == 'list-compilers':
 535             for name in sorted(self.configs.keys()):
 536                 print(name)
 537             return
 538         if action == 'list-glibcs':
 539             for config in sorted(self.glibc_configs.values(),
 540                                  key=lambda c: c.name):
 541                 print(config.name, config.compiler.name)
 542             return
 543         self.clear_last_build_state(action)
 544         build_time = datetime.datetime.utcnow()
 545         if action == 'host-libraries':
 546             build_components = ('gmp', 'mpfr', 'mpc')
 547             old_components = ()
 548             old_versions = {}
 549             self.build_host_libraries()
 550         elif action == 'compilers':
 551             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 552                                 'gnumach', 'hurd')
 553             old_components = ('gmp', 'mpfr', 'mpc')
 554             old_versions = self.build_state['host-libraries']['build-versions']
 555             self.build_compilers(configs)
 556         else:
 557             build_components = ('glibc',)
 558             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 559                               'mig', 'gnumach', 'hurd')
 560             old_versions = self.build_state['compilers']['build-versions']
 561             if action == 'update-syscalls':
 562                 self.update_syscalls(configs)
 563             else:
 564                 self.build_glibcs(configs)
 565         self.write_files()
 566         self.do_build()
 567         if configs:
 568             # Partial build, do not update stored state.
 569             return
 570         build_versions = {}
 571         for k in build_components:
 572             if k in self.versions:
 573                 build_versions[k] = {'version': self.versions[k]['version'],
 574                                      'revision': self.versions[k]['revision']}
 575         for k in old_components:
 576             if k in old_versions:
 577                 build_versions[k] = {'version': old_versions[k]['version'],
 578                                      'revision': old_versions[k]['revision']}
 579         self.update_build_state(action, build_time, build_versions)
 580
 581     @staticmethod
 582     def remove_dirs(*args):
 583         """Remove directories and their contents if they exist."""
 584         for dir in args:
 585             shutil.rmtree(dir, ignore_errors=True)
 586
 587     @staticmethod
 588     def remove_recreate_dirs(*args):
 589         """Remove directories if they exist, and create them as empty."""
 590         Context.remove_dirs(*args)
 591         for dir in args:
 592             os.makedirs(dir, exist_ok=True)
 593
 594     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 595         """Add makefile text for a list of commands."""
 596         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 597         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 598                                     (target, target, target, commands))
 599         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 600
 601     def write_files(self):
 602         """Write out the Makefile and wrapper script."""
 603         mftext = ''.join(self.makefile_pieces)
 604         with open(self.makefile, 'w') as f:
 605             f.write(mftext)
 606         wrapper_text = (
 607             '#!/bin/sh\n'
 608             'prev_base=$1\n'
 609             'this_base=$2\n'
 610             'desc=$3\n'
 611             'dir=$4\n'
 612             'path=$5\n'
 613             'shift 5\n'
 614             'prev_status=$prev_base-status.txt\n'
 615             'this_status=$this_base-status.txt\n'
 616             'this_log=$this_base-log.txt\n'
 617             'date > "$this_log"\n'
 618             'echo >> "$this_log"\n'
 619             'echo "Description: $desc" >> "$this_log"\n'
 620             'printf "%s" "Command:" >> "$this_log"\n'
 621             'for word in "$@"; do\n'
 622             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 623             '    printf " %s" "$word"\n'
 624             '  else\n'
 625             '    printf " \'"\n'
 626             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 627             '    printf "\'"\n'
 628             '  fi\n'
 629             'done >> "$this_log"\n'
 630             'echo >> "$this_log"\n'
 631             'echo "Directory: $dir" >> "$this_log"\n'
 632             'echo "Path addition: $path" >> "$this_log"\n'
 633             'echo >> "$this_log"\n'
 634             'record_status ()\n'
 635             '{\n'
 636             '  echo >> "$this_log"\n'
 637             '  echo "$1: $desc" > "$this_status"\n'
 638             '  echo "$1: $desc" >> "$this_log"\n'
 639             '  echo >> "$this_log"\n'
 640             '  date >> "$this_log"\n'
 641             '  echo "$1: $desc"\n'
 642             '  exit 0\n'
 643             '}\n'
 644             'check_error ()\n'
 645             '{\n'
 646             '  if [ "$1" != "0" ]; then\n'
 647             '    record_status FAIL\n'
 648             '  fi\n'
 649             '}\n'
 650             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 651             '    record_status UNRESOLVED\n'
 652             'fi\n'
 653             'if [ "$dir" ]; then\n'
 654             '  cd "$dir"\n'
 655             '  check_error "$?"\n'
 656             'fi\n'
 657             'if [ "$path" ]; then\n'
 658             '  PATH=$path:$PATH\n'
 659             'fi\n'
 660             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 661             'check_error "$?"\n'
 662             'record_status PASS\n')
 663         with open(self.wrapper, 'w') as f:
 664             f.write(wrapper_text)
 665         # Mode 0o755.
 666         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 667                      stat.S_IROTH|stat.S_IXOTH)
 668         os.chmod(self.wrapper, mode_exec)
 669         save_logs_text = (
 670             '#!/bin/sh\n'
 671             'if ! [ -f tests.sum ]; then\n'
 672             '  echo "No test summary available."\n'
 673             '  exit 0\n'
 674             'fi\n'
 675             'save_file ()\n'
 676             '{\n'
 677             '  echo "Contents of $1:"\n'
 678             '  echo\n'
 679             '  cat "$1"\n'
 680             '  echo\n'
 681             '  echo "End of contents of $1."\n'
 682             '  echo\n'
 683             '}\n'
 684             'save_file tests.sum\n'
 685             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 686             'for t in $non_pass_tests; do\n'
 687             '  if [ -f "$t.out" ]; then\n'
 688             '    save_file "$t.out"\n'
 689             '  fi\n'
 690             'done\n')
 691         with open(self.save_logs, 'w') as f:
 692             f.write(save_logs_text)
 693         os.chmod(self.save_logs, mode_exec)
 694
 695     def do_build(self):
 696         """Do the actual build."""
 697         cmd = ['make', '-O', '-j%d' % self.parallelism]
 698         subprocess.run(cmd, cwd=self.builddir, check=True)
 699
 700     def build_host_libraries(self):
 701         """Build the host libraries."""
 702         installdir = self.host_libraries_installdir
 703         builddir = os.path.join(self.builddir, 'host-libraries')
 704         logsdir = os.path.join(self.logsdir, 'host-libraries')
 705         self.remove_recreate_dirs(installdir, builddir, logsdir)
 706         cmdlist = CommandList('host-libraries', self.keep)
 707         self.build_host_library(cmdlist, 'gmp')
 708         self.build_host_library(cmdlist, 'mpfr',
 709                                 ['--with-gmp=%s' % installdir])
 710         self.build_host_library(cmdlist, 'mpc',
 711                                 ['--with-gmp=%s' % installdir,
 712                                 '--with-mpfr=%s' % installdir])
 713         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 714         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 715
 716     def build_host_library(self, cmdlist, lib, extra_opts=None):
 717         """Build one host library."""
 718         srcdir = self.component_srcdir(lib)
 719         builddir = self.component_builddir('host-libraries', None, lib)
 720         installdir = self.host_libraries_installdir
 721         cmdlist.push_subdesc(lib)
 722         cmdlist.create_use_dir(builddir)
 723         cfg_cmd = [os.path.join(srcdir, 'configure'),
 724                    '--prefix=%s' % installdir,
 725                    '--disable-shared']
 726         if extra_opts:
 727             cfg_cmd.extend (extra_opts)
 728         cmdlist.add_command('configure', cfg_cmd)
 729         cmdlist.add_command('build', ['make'])
 730         cmdlist.add_command('check', ['make', 'check'])
 731         cmdlist.add_command('install', ['make', 'install'])
 732         cmdlist.cleanup_dir()
 733         cmdlist.pop_subdesc()
 734
 735     def build_compilers(self, configs):
 736         """Build the compilers."""
 737         if not configs:
 738             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 739             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 740             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 741             configs = sorted(self.configs.keys())
 742         for c in configs:
 743             self.configs[c].build()
 744
 745     def build_glibcs(self, configs):
 746         """Build the glibcs."""
 747         if not configs:
 748             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 749             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 750             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 751             configs = sorted(self.glibc_configs.keys())
 752         for c in configs:
 753             self.glibc_configs[c].build()
 754
 755     def update_syscalls(self, configs):
 756         """Update the glibc syscall lists."""
 757         if not configs:
 758             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 759             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 760             configs = sorted(self.glibc_configs.keys())
 761         for c in configs:
 762             self.glibc_configs[c].update_syscalls()
 763
 764     def load_versions_json(self):
 765         """Load information about source directory versions."""
 766         if not os.access(self.versions_json, os.F_OK):
 767             self.versions = {}
 768             return
 769         with open(self.versions_json, 'r') as f:
 770             self.versions = json.load(f)
 771
 772     def store_json(self, data, filename):
 773         """Store information in a JSON file."""
 774         filename_tmp = filename + '.tmp'
 775         with open(filename_tmp, 'w') as f:
 776             json.dump(data, f, indent=2, sort_keys=True)
 777         os.rename(filename_tmp, filename)
 778
 779     def store_versions_json(self):
 780         """Store information about source directory versions."""
 781         self.store_json(self.versions, self.versions_json)
 782
 783     def set_component_version(self, component, version, explicit, revision):
 784         """Set the version information for a component."""
 785         self.versions[component] = {'version': version,
 786                                     'explicit': explicit,
 787                                     'revision': revision}
 788         self.store_versions_json()
 789
 790     def checkout(self, versions):
 791         """Check out the desired component versions."""
 792         default_versions = {'binutils': 'vcs-2.38',
 793                             'gcc': 'vcs-12',
 794                             'glibc': 'vcs-mainline',
 795                             'gmp': '6.2.1',
 796                             'linux': '5.18',
 797                             'mpc': '1.2.1',
 798                             'mpfr': '4.1.0',
 799                             'mig': 'vcs-mainline',
 800                             'gnumach': 'vcs-mainline',
 801                             'hurd': 'vcs-mainline'}
 802         use_versions = {}
 803         explicit_versions = {}
 804         for v in versions:
 805             found_v = False
 806             for k in default_versions.keys():
 807                 kx = k + '-'
 808                 if v.startswith(kx):
 809                     vx = v[len(kx):]
 810                     if k in use_versions:
 811                         print('error: multiple versions for %s' % k)
 812                         exit(1)
 813                     use_versions[k] = vx
 814                     explicit_versions[k] = True
 815                     found_v = True
 816                     break
 817             if not found_v:
 818                 print('error: unknown component in %s' % v)
 819                 exit(1)
 820         for k in default_versions.keys():
 821             if k not in use_versions:
 822                 if k in self.versions and self.versions[k]['explicit']:
 823                     use_versions[k] = self.versions[k]['version']
 824                     explicit_versions[k] = True
 825                 else:
 826                     use_versions[k] = default_versions[k]
 827                     explicit_versions[k] = False
 828         os.makedirs(self.srcdir, exist_ok=True)
 829         for k in sorted(default_versions.keys()):
 830             update = os.access(self.component_srcdir(k), os.F_OK)
 831             v = use_versions[k]
 832             if (update and
 833                 k in self.versions and
 834                 v != self.versions[k]['version']):
 835                 if not self.replace_sources:
 836                     print('error: version of %s has changed from %s to %s, '
 837                           'use --replace-sources to check out again' %
 838                           (k, self.versions[k]['version'], v))
 839                     exit(1)
 840                 shutil.rmtree(self.component_srcdir(k))
 841                 update = False
 842             if v.startswith('vcs-'):
 843                 revision = self.checkout_vcs(k, v[4:], update)
 844             else:
 845                 self.checkout_tar(k, v, update)
 846                 revision = v
 847             self.set_component_version(k, v, explicit_versions[k], revision)
 848         if self.get_script_text() != self.script_text:
 849             # Rerun the checkout process in case the updated script
 850             # uses different default versions or new components.
 851             self.exec_self()
 852
 853     def checkout_vcs(self, component, version, update):
 854         """Check out the given version of the given component from version
 855         control.  Return a revision identifier."""
 856         if component == 'binutils':
 857             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 858             if version == 'mainline':
 859                 git_branch = 'master'
 860             else:
 861                 trans = str.maketrans({'.': '_'})
 862                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 863             return self.git_checkout(component, git_url, git_branch, update)
 864         elif component == 'gcc':
 865             if version == 'mainline':
 866                 branch = 'master'
 867             else:
 868                 branch = 'releases/gcc-%s' % version
 869             return self.gcc_checkout(branch, update)
 870         elif component == 'glibc':
 871             git_url = 'git://sourceware.org/git/glibc.git'
 872             if version == 'mainline':
 873                 git_branch = 'master'
 874             else:
 875                 git_branch = 'release/%s/master' % version
 876             r = self.git_checkout(component, git_url, git_branch, update)
 877             self.fix_glibc_timestamps()
 878             return r
 879         elif component == 'gnumach':
 880             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 881             git_branch = 'master'
 882             r = self.git_checkout(component, git_url, git_branch, update)
 883             subprocess.run(['autoreconf', '-i'],
 884                            cwd=self.component_srcdir(component), check=True)
 885             return r
 886         elif component == 'mig':
 887             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 888             git_branch = 'master'
 889             r = self.git_checkout(component, git_url, git_branch, update)
 890             subprocess.run(['autoreconf', '-i'],
 891                            cwd=self.component_srcdir(component), check=True)
 892             return r
 893         elif component == 'hurd':
 894             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 895             git_branch = 'master'
 896             r = self.git_checkout(component, git_url, git_branch, update)
 897             subprocess.run(['autoconf'],
 898                            cwd=self.component_srcdir(component), check=True)
 899             return r
 900         else:
 901             print('error: component %s coming from VCS' % component)
 902             exit(1)
 903
 904     def git_checkout(self, component, git_url, git_branch, update):
 905         """Check out a component from git.  Return a commit identifier."""
 906         if update:
 907             subprocess.run(['git', 'remote', 'prune', 'origin'],
 908                            cwd=self.component_srcdir(component), check=True)
 909             if self.replace_sources:
 910                 subprocess.run(['git', 'clean', '-dxfq'],
 911                                cwd=self.component_srcdir(component), check=True)
 912             subprocess.run(['git', 'pull', '-q'],
 913                            cwd=self.component_srcdir(component), check=True)
 914         else:
 915             if self.shallow:
 916                 depth_arg = ('--depth', '1')
 917             else:
 918                 depth_arg = ()
 919             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 920                             *depth_arg, git_url,
 921                             self.component_srcdir(component)], check=True)
 922         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 923                            cwd=self.component_srcdir(component),
 924                            stdout=subprocess.PIPE,
 925                            check=True, universal_newlines=True).stdout
 926         return r.rstrip()
 927
 928     def fix_glibc_timestamps(self):
 929         """Fix timestamps in a glibc checkout."""
 930         # Ensure that builds do not try to regenerate generated files
 931         # in the source tree.
 932         srcdir = self.component_srcdir('glibc')
 933         # These files have Makefile dependencies to regenerate them in
 934         # the source tree that may be active during a normal build.
 935         # Some other files have such dependencies but do not need to
 936         # be touched because nothing in a build depends on the files
 937         # in question.
 938         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 939             to_touch = os.path.join(srcdir, f)
 940             subprocess.run(['touch', '-c', to_touch], check=True)
 941         for dirpath, dirnames, filenames in os.walk(srcdir):
 942             for f in filenames:
 943                 if (f == 'configure' or
 944                     f == 'preconfigure' or
 945                     f.endswith('-kw.h')):
 946                     to_touch = os.path.join(dirpath, f)
 947                     subprocess.run(['touch', to_touch], check=True)
 948
 949     def gcc_checkout(self, branch, update):
 950         """Check out GCC from git.  Return the commit identifier."""
 951         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 952                      os.F_OK):
 953             if not self.replace_sources:
 954                 print('error: GCC has moved from SVN to git, use '
 955                       '--replace-sources to check out again')
 956                 exit(1)
 957             shutil.rmtree(self.component_srcdir('gcc'))
 958             update = False
 959         if not update:
 960             self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
 961                               branch, update)
 962         subprocess.run(['contrib/gcc_update', '--silent'],
 963                        cwd=self.component_srcdir('gcc'), check=True)
 964         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 965                            cwd=self.component_srcdir('gcc'),
 966                            stdout=subprocess.PIPE,
 967                            check=True, universal_newlines=True).stdout
 968         return r.rstrip()
 969
 970     def checkout_tar(self, component, version, update):
 971         """Check out the given version of the given component from a
 972         tarball."""
 973         if update:
 974             return
 975         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 976                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 977                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 978                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 979                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 980                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 981                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 982                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 983                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 984         if component not in url_map:
 985             print('error: component %s coming from tarball' % component)
 986             exit(1)
 987         version_major = version.split('.')[0]
 988         url = url_map[component] % {'version': version, 'major': version_major}
 989         filename = os.path.join(self.srcdir, url.split('/')[-1])
 990         response = urllib.request.urlopen(url)
 991         data = response.read()
 992         with open(filename, 'wb') as f:
 993             f.write(data)
 994         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 995                        check=True)
 996         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 997                   self.component_srcdir(component))
 998         os.remove(filename)
 999
1000     def load_build_state_json(self):
1001         """Load information about the state of previous builds."""
1002         if os.access(self.build_state_json, os.F_OK):
1003             with open(self.build_state_json, 'r') as f:
1004                 self.build_state = json.load(f)
1005         else:
1006             self.build_state = {}
1007         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1008             if k not in self.build_state:
1009                 self.build_state[k] = {}
1010             if 'build-time' not in self.build_state[k]:
1011                 self.build_state[k]['build-time'] = ''
1012             if 'build-versions' not in self.build_state[k]:
1013                 self.build_state[k]['build-versions'] = {}
1014             if 'build-results' not in self.build_state[k]:
1015                 self.build_state[k]['build-results'] = {}
1016             if 'result-changes' not in self.build_state[k]:
1017                 self.build_state[k]['result-changes'] = {}
1018             if 'ever-passed' not in self.build_state[k]:
1019                 self.build_state[k]['ever-passed'] = []
1020
1021     def store_build_state_json(self):
1022         """Store information about the state of previous builds."""
1023         self.store_json(self.build_state, self.build_state_json)
1024
1025     def clear_last_build_state(self, action):
1026         """Clear information about the state of part of the build."""
1027         # We clear the last build time and versions when starting a
1028         # new build.  The results of the last build are kept around,
1029         # as comparison is still meaningful if this build is aborted
1030         # and a new one started.
1031         self.build_state[action]['build-time'] = ''
1032         self.build_state[action]['build-versions'] = {}
1033         self.store_build_state_json()
1034
1035     def update_build_state(self, action, build_time, build_versions):
1036         """Update the build state after a build."""
1037         build_time = build_time.replace(microsecond=0)
1038         self.build_state[action]['build-time'] = str(build_time)
1039         self.build_state[action]['build-versions'] = build_versions
1040         build_results = {}
1041         for log in self.status_log_list:
1042             with open(log, 'r') as f:
1043                 log_text = f.read()
1044             log_text = log_text.rstrip()
1045             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1046             result = m.group(1)
1047             test_name = m.group(2)
1048             assert test_name not in build_results
1049             build_results[test_name] = result
1050         old_build_results = self.build_state[action]['build-results']
1051         self.build_state[action]['build-results'] = build_results
1052         result_changes = {}
1053         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1054         for t in all_tests:
1055             if t in old_build_results:
1056                 old_res = old_build_results[t]
1057             else:
1058                 old_res = '(New test)'
1059             if t in build_results:
1060                 new_res = build_results[t]
1061             else:
1062                 new_res = '(Test removed)'
1063             if old_res != new_res:
1064                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1065         self.build_state[action]['result-changes'] = result_changes
1066         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1067                            if t in build_results}
1068         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1069         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1070                                                          new_passes)
1071         self.store_build_state_json()
1072
1073     def load_bot_config_json(self):
1074         """Load bot configuration."""
1075         with open(self.bot_config_json, 'r') as f:
1076             self.bot_config = json.load(f)
1077
1078     def part_build_old(self, action, delay):
1079         """Return whether the last build for a given action was at least a
1080         given number of seconds ago, or does not have a time recorded."""
1081         old_time_str = self.build_state[action]['build-time']
1082         if not old_time_str:
1083             return True
1084         old_time = datetime.datetime.strptime(old_time_str,
1085                                               '%Y-%m-%d %H:%M:%S')
1086         new_time = datetime.datetime.utcnow()
1087         delta = new_time - old_time
1088         return delta.total_seconds() >= delay
1089
1090     def bot_cycle(self):
1091         """Run a single round of checkout and builds."""
1092         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1093         self.load_bot_config_json()
1094         actions = ('host-libraries', 'compilers', 'glibcs')
1095         self.bot_run_self(['--replace-sources'], 'checkout')
1096         self.load_versions_json()
1097         if self.get_script_text() != self.script_text:
1098             print('Script changed, re-execing.')
1099             # On script change, all parts of the build should be rerun.
1100             for a in actions:
1101                 self.clear_last_build_state(a)
1102             self.exec_self()
1103         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1104                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1105                                           'mig', 'gnumach', 'hurd'),
1106                             'glibcs': ('glibc',)}
1107         must_build = {}
1108         for a in actions:
1109             build_vers = self.build_state[a]['build-versions']
1110             must_build[a] = False
1111             if not self.build_state[a]['build-time']:
1112                 must_build[a] = True
1113             old_vers = {}
1114             new_vers = {}
1115             for c in check_components[a]:
1116                 if c in build_vers:
1117                     old_vers[c] = build_vers[c]
1118                 new_vers[c] = {'version': self.versions[c]['version'],
1119                                'revision': self.versions[c]['revision']}
1120             if new_vers == old_vers:
1121                 print('Versions for %s unchanged.' % a)
1122             else:
1123                 print('Versions changed or rebuild forced for %s.' % a)
1124                 if a == 'compilers' and not self.part_build_old(
1125                         a, self.bot_config['compilers-rebuild-delay']):
1126                     print('Not requiring rebuild of compilers this soon.')
1127                 else:
1128                     must_build[a] = True
1129         if must_build['host-libraries']:
1130             must_build['compilers'] = True
1131         if must_build['compilers']:
1132             must_build['glibcs'] = True
1133         for a in actions:
1134             if must_build[a]:
1135                 print('Must rebuild %s.' % a)
1136                 self.clear_last_build_state(a)
1137             else:
1138                 print('No need to rebuild %s.' % a)
1139         if os.access(self.logsdir, os.F_OK):
1140             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1141             shutil.copytree(self.logsdir, self.logsdir_old)
1142         for a in actions:
1143             if must_build[a]:
1144                 build_time = datetime.datetime.utcnow()
1145                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1146                 self.bot_run_self([], a)
1147                 self.load_build_state_json()
1148                 self.bot_build_mail(a, build_time)
1149         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1150
1151     def bot_build_mail(self, action, build_time):
1152         """Send email with the results of a build."""
1153         if not ('email-from' in self.bot_config and
1154                 'email-server' in self.bot_config and
1155                 'email-subject' in self.bot_config and
1156                 'email-to' in self.bot_config):
1157             if not self.email_warning:
1158                 print("Email not configured, not sending.")
1159                 self.email_warning = True
1160             return
1161
1162         build_time = build_time.replace(microsecond=0)
1163         subject = (self.bot_config['email-subject'] %
1164                    {'action': action,
1165                     'build-time': str(build_time)})
1166         results = self.build_state[action]['build-results']
1167         changes = self.build_state[action]['result-changes']
1168         ever_passed = set(self.build_state[action]['ever-passed'])
1169         versions = self.build_state[action]['build-versions']
1170         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1171         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1172         all_fails = {k for k in results if results[k] == 'FAIL'}
1173         if new_regressions:
1174             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1175             new_reg_text = ('New regressions:\n\n%s\n\n' %
1176                             '\n'.join(new_reg_list))
1177         else:
1178             new_reg_text = ''
1179         if all_regressions:
1180             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1181             all_reg_text = ('All regressions:\n\n%s\n\n' %
1182                             '\n'.join(all_reg_list))
1183         else:
1184             all_reg_text = ''
1185         if all_fails:
1186             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1187             all_fail_text = ('All failures:\n\n%s\n\n' %
1188                              '\n'.join(all_fail_list))
1189         else:
1190             all_fail_text = ''
1191         if changes:
1192             changes_list = sorted(changes.keys())
1193             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1194             changes_text = ('All changed results:\n\n%s\n\n' %
1195                             '\n'.join(changes_list))
1196         else:
1197             changes_text = ''
1198         results_text = (new_reg_text + all_reg_text + all_fail_text +
1199                         changes_text)
1200         if not results_text:
1201             results_text = 'Clean build with unchanged results.\n\n'
1202         versions_list = sorted(versions.keys())
1203         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1204                                           versions[k]['revision'])
1205                          for k in versions_list]
1206         versions_text = ('Component versions for this build:\n\n%s\n' %
1207                          '\n'.join(versions_list))
1208         body_text = results_text + versions_text
1209         msg = email.mime.text.MIMEText(body_text)
1210         msg['Subject'] = subject
1211         msg['From'] = self.bot_config['email-from']
1212         msg['To'] = self.bot_config['email-to']
1213         msg['Message-ID'] = email.utils.make_msgid()
1214         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1215         with smtplib.SMTP(self.bot_config['email-server']) as s:
1216             s.send_message(msg)
1217
1218     def bot_run_self(self, opts, action, check=True):
1219         """Run a copy of this script with given options."""
1220         cmd = [sys.executable, sys.argv[0], '--keep=none',
1221                '-j%d' % self.parallelism]
1222         if self.full_gcc:
1223             cmd.append('--full-gcc')
1224         cmd.extend(opts)
1225         cmd.extend([self.topdir, action])
1226         sys.stdout.flush()
1227         subprocess.run(cmd, check=check)
1228
1229     def bot(self):
1230         """Run repeated rounds of checkout and builds."""
1231         while True:
1232             self.load_bot_config_json()
1233             if not self.bot_config['run']:
1234                 print('Bot exiting by request.')
1235                 exit(0)
1236             self.bot_run_self([], 'bot-cycle', check=False)
1237             self.load_bot_config_json()
1238             if not self.bot_config['run']:
1239                 print('Bot exiting by request.')
1240                 exit(0)
1241             time.sleep(self.bot_config['delay'])
1242             if self.get_script_text() != self.script_text:
1243                 print('Script changed, bot re-execing.')
1244                 self.exec_self()
1245
1246 class LinuxHeadersPolicyForBuild(object):
1247     """Names and directories for installing Linux headers.  Build variant."""
1248
1249     def __init__(self, config):
1250         self.arch = config.arch
1251         self.srcdir = config.ctx.component_srcdir('linux')
1252         self.builddir = config.component_builddir('linux')
1253         self.headers_dir = os.path.join(config.sysroot, 'usr')
1254
1255 class LinuxHeadersPolicyForUpdateSyscalls(object):
1256     """Names and directories for Linux headers.  update-syscalls variant."""
1257
1258     def __init__(self, glibc, headers_dir):
1259         self.arch = glibc.compiler.arch
1260         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1261         self.builddir = glibc.ctx.component_builddir(
1262             'update-syscalls', glibc.name, 'build-linux')
1263         self.headers_dir = headers_dir
1264
1265 def install_linux_headers(policy, cmdlist):
1266     """Install Linux kernel headers."""
1267     arch_map = {'aarch64': 'arm64',
1268                 'alpha': 'alpha',
1269                 'arc': 'arc',
1270                 'arm': 'arm',
1271                 'csky': 'csky',
1272                 'hppa': 'parisc',
1273                 'i486': 'x86',
1274                 'i586': 'x86',
1275                 'i686': 'x86',
1276                 'i786': 'x86',
1277                 'ia64': 'ia64',
1278                 'loongarch64': 'loongarch',
1279                 'm68k': 'm68k',
1280                 'microblaze': 'microblaze',
1281                 'mips': 'mips',
1282                 'nios2': 'nios2',
1283                 'or1k': 'openrisc',
1284                 'powerpc': 'powerpc',
1285                 's390': 's390',
1286                 'riscv32': 'riscv',
1287                 'riscv64': 'riscv',
1288                 'sh': 'sh',
1289                 'sparc': 'sparc',
1290                 'x86_64': 'x86'}
1291     linux_arch = None
1292     for k in arch_map:
1293         if policy.arch.startswith(k):
1294             linux_arch = arch_map[k]
1295             break
1296     assert linux_arch is not None
1297     cmdlist.push_subdesc('linux')
1298     cmdlist.create_use_dir(policy.builddir)
1299     cmdlist.add_command('install-headers',
1300                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1301                          'ARCH=%s' % linux_arch,
1302                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1303                          'headers_install'])
1304     cmdlist.cleanup_dir()
1305     cmdlist.pop_subdesc()
1306
1307 class Config(object):
1308     """A configuration for building a compiler and associated libraries."""
1309
1310     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1311                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1312                  extra_glibcs=None):
1313         """Initialize a Config object."""
1314         self.ctx = ctx
1315         self.arch = arch
1316         self.os = os_name
1317         self.variant = variant
1318         if variant is None:
1319             self.name = '%s-%s' % (arch, os_name)
1320         else:
1321             self.name = '%s-%s-%s' % (arch, os_name, variant)
1322         self.triplet = '%s-glibc-%s' % (arch, os_name)
1323         if gcc_cfg is None:
1324             self.gcc_cfg = []
1325         else:
1326             self.gcc_cfg = gcc_cfg
1327         if first_gcc_cfg is None:
1328             self.first_gcc_cfg = []
1329         else:
1330             self.first_gcc_cfg = first_gcc_cfg
1331         if binutils_cfg is None:
1332             self.binutils_cfg = []
1333         else:
1334             self.binutils_cfg = binutils_cfg
1335         if glibcs is None:
1336             glibcs = [{'variant': variant}]
1337         if extra_glibcs is None:
1338             extra_glibcs = []
1339         glibcs = [Glibc(self, **g) for g in glibcs]
1340         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1341         self.all_glibcs = glibcs + extra_glibcs
1342         self.compiler_glibcs = glibcs
1343         self.installdir = ctx.compiler_installdir(self.name)
1344         self.bindir = ctx.compiler_bindir(self.name)
1345         self.sysroot = ctx.compiler_sysroot(self.name)
1346         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1347         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1348
1349     def component_builddir(self, component):
1350         """Return the directory to use for a (non-glibc) build."""
1351         return self.ctx.component_builddir('compilers', self.name, component)
1352
1353     def build(self):
1354         """Generate commands to build this compiler."""
1355         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1356                                       self.logsdir)
1357         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1358         cmdlist.add_command('check-host-libraries',
1359                             ['test', '-f',
1360                              os.path.join(self.ctx.host_libraries_installdir,
1361                                           'ok')])
1362         cmdlist.use_path(self.bindir)
1363         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1364                               ['--disable-gdb',
1365                                '--disable-gdbserver',
1366                                '--disable-libdecnumber',
1367                                '--disable-readline',
1368                                '--disable-sim'] + self.binutils_cfg)
1369         if self.os.startswith('linux'):
1370             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1371         self.build_gcc(cmdlist, True)
1372         if self.os == 'gnu':
1373             self.install_gnumach_headers(cmdlist)
1374             self.build_cross_tool(cmdlist, 'mig', 'mig')
1375             self.install_hurd_headers(cmdlist)
1376         for g in self.compiler_glibcs:
1377             cmdlist.push_subdesc('glibc')
1378             cmdlist.push_subdesc(g.name)
1379             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1380             cmdlist.pop_subdesc()
1381             cmdlist.pop_subdesc()
1382         self.build_gcc(cmdlist, False)
1383         cmdlist.add_command('done', ['touch',
1384                                      os.path.join(self.installdir, 'ok')])
1385         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1386                                       self.logsdir)
1387
1388     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1389         """Build one cross tool."""
1390         srcdir = self.ctx.component_srcdir(tool_src)
1391         builddir = self.component_builddir(tool_build)
1392         cmdlist.push_subdesc(tool_build)
1393         cmdlist.create_use_dir(builddir)
1394         cfg_cmd = [os.path.join(srcdir, 'configure'),
1395                    '--prefix=%s' % self.installdir,
1396                    '--build=%s' % self.ctx.build_triplet,
1397                    '--host=%s' % self.ctx.build_triplet,
1398                    '--target=%s' % self.triplet,
1399                    '--with-sysroot=%s' % self.sysroot]
1400         if extra_opts:
1401             cfg_cmd.extend(extra_opts)
1402         cmdlist.add_command('configure', cfg_cmd)
1403         cmdlist.add_command('build', ['make'])
1404         # Parallel "make install" for GCC has race conditions that can
1405         # cause it to fail; see
1406         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1407         # problems are not known for binutils, but doing the
1408         # installation in parallel within a particular toolchain build
1409         # (as opposed to installation of one toolchain from
1410         # build-many-glibcs.py running in parallel to the installation
1411         # of other toolchains being built) is not known to be
1412         # significantly beneficial, so it is simplest just to disable
1413         # parallel install for cross tools here.
1414         cmdlist.add_command('install', ['make', '-j1', 'install'])
1415         cmdlist.cleanup_dir()
1416         cmdlist.pop_subdesc()
1417
1418     def install_gnumach_headers(self, cmdlist):
1419         """Install GNU Mach headers."""
1420         srcdir = self.ctx.component_srcdir('gnumach')
1421         builddir = self.component_builddir('gnumach')
1422         cmdlist.push_subdesc('gnumach')
1423         cmdlist.create_use_dir(builddir)
1424         cmdlist.add_command('configure',
1425                             [os.path.join(srcdir, 'configure'),
1426                              '--build=%s' % self.ctx.build_triplet,
1427                              '--host=%s' % self.triplet,
1428                              '--prefix=',
1429                              'CC=%s-gcc -nostdlib' % self.triplet])
1430         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1431                                         'install-data'])
1432         cmdlist.cleanup_dir()
1433         cmdlist.pop_subdesc()
1434
1435     def install_hurd_headers(self, cmdlist):
1436         """Install Hurd headers."""
1437         srcdir = self.ctx.component_srcdir('hurd')
1438         builddir = self.component_builddir('hurd')
1439         cmdlist.push_subdesc('hurd')
1440         cmdlist.create_use_dir(builddir)
1441         cmdlist.add_command('configure',
1442                             [os.path.join(srcdir, 'configure'),
1443                              '--build=%s' % self.ctx.build_triplet,
1444                              '--host=%s' % self.triplet,
1445                              '--prefix=',
1446                              '--disable-profile', '--without-parted',
1447                              'CC=%s-gcc -nostdlib' % self.triplet])
1448         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1449                                         'no_deps=t', 'install-headers'])
1450         cmdlist.cleanup_dir()
1451         cmdlist.pop_subdesc()
1452
1453     def build_gcc(self, cmdlist, bootstrap):
1454         """Build GCC."""
1455         # libssp is of little relevance with glibc's own stack
1456         # checking support.  libcilkrts does not support GNU/Hurd (and
1457         # has been removed in GCC 8, so --disable-libcilkrts can be
1458         # removed once glibc no longer supports building with older
1459         # GCC versions).  --enable-initfini-array is enabled by default
1460         # in GCC 12, which can be removed when GCC 12 becomes the
1461         # minimum requirement.
1462         cfg_opts = list(self.gcc_cfg)
1463         cfg_opts += ['--enable-initfini-array']
1464         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1465         host_libs = self.ctx.host_libraries_installdir
1466         cfg_opts += ['--with-gmp=%s' % host_libs,
1467                      '--with-mpfr=%s' % host_libs,
1468                      '--with-mpc=%s' % host_libs]
1469         if bootstrap:
1470             tool_build = 'gcc-first'
1471             # Building a static-only, C-only compiler that is
1472             # sufficient to build glibc.  Various libraries and
1473             # features that may require libc headers must be disabled.
1474             # When configuring with a sysroot, --with-newlib is
1475             # required to define inhibit_libc (to stop some parts of
1476             # libgcc including libc headers); --without-headers is not
1477             # sufficient.
1478             cfg_opts += ['--enable-languages=c', '--disable-shared',
1479                          '--disable-threads',
1480                          '--disable-libatomic',
1481                          '--disable-decimal-float',
1482                          '--disable-libffi',
1483                          '--disable-libgomp',
1484                          '--disable-libitm',
1485                          '--disable-libmpx',
1486                          '--disable-libquadmath',
1487                          '--disable-libsanitizer',
1488                          '--without-headers', '--with-newlib',
1489                          '--with-glibc-version=%s' % self.ctx.glibc_version
1490                          ]
1491             cfg_opts += self.first_gcc_cfg
1492         else:
1493             tool_build = 'gcc'
1494             # libsanitizer commonly breaks because of glibc header
1495             # changes, or on unusual targets.  C++ pre-compiled
1496             # headers are not used during the glibc build and are
1497             # expensive to create.
1498             if not self.ctx.full_gcc:
1499                 cfg_opts += ['--disable-libsanitizer',
1500                              '--disable-libstdcxx-pch']
1501             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1502             cfg_opts += ['--enable-languages=%s' % langs,
1503                          '--enable-shared', '--enable-threads']
1504         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1505
1506 class GlibcPolicyDefault(object):
1507     """Build policy for glibc: common defaults."""
1508
1509     def __init__(self, glibc):
1510         self.srcdir = glibc.ctx.component_srcdir('glibc')
1511         self.use_usr = glibc.os != 'gnu'
1512         self.prefix = '/usr' if self.use_usr else ''
1513         self.configure_args = [
1514             '--prefix=%s' % self.prefix,
1515             '--enable-profile',
1516             '--build=%s' % glibc.ctx.build_triplet,
1517             '--host=%s' % glibc.triplet,
1518             'CC=%s' % glibc.tool_name('gcc'),
1519             'CXX=%s' % glibc.tool_name('g++'),
1520             'AR=%s' % glibc.tool_name('ar'),
1521             'AS=%s' % glibc.tool_name('as'),
1522             'LD=%s' % glibc.tool_name('ld'),
1523             'NM=%s' % glibc.tool_name('nm'),
1524             'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1525             'OBJDUMP=%s' % glibc.tool_name('objdump'),
1526             'RANLIB=%s' % glibc.tool_name('ranlib'),
1527             'READELF=%s' % glibc.tool_name('readelf'),
1528             'STRIP=%s' % glibc.tool_name('strip'),
1529         ]
1530         if glibc.os == 'gnu':
1531             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1532         if glibc.cflags:
1533             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1534             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1535         self.configure_args += glibc.cfg
1536
1537     def configure(self, cmdlist):
1538         """Invoked to add the configure command to the command list."""
1539         cmdlist.add_command('configure',
1540                             [os.path.join(self.srcdir, 'configure'),
1541                              *self.configure_args])
1542
1543     def extra_commands(self, cmdlist):
1544         """Invoked to inject additional commands (make check) after build."""
1545         pass
1546
1547 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1548     """Build policy for glibc during the compilers stage."""
1549
1550     def __init__(self, glibc):
1551         super().__init__(glibc)
1552         self.builddir = glibc.ctx.component_builddir(
1553             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1554         self.installdir = glibc.compiler.sysroot
1555
1556 class GlibcPolicyForBuild(GlibcPolicyDefault):
1557     """Build policy for glibc during the glibcs stage."""
1558
1559     def __init__(self, glibc):
1560         super().__init__(glibc)
1561         self.builddir = glibc.ctx.component_builddir(
1562             'glibcs', glibc.name, 'glibc')
1563         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1564         if glibc.ctx.strip:
1565             self.strip = glibc.tool_name('strip')
1566         else:
1567             self.strip = None
1568         self.save_logs = glibc.ctx.save_logs
1569
1570     def extra_commands(self, cmdlist):
1571         if self.strip:
1572             # Avoid stripping libc.so and libpthread.so, which are
1573             # linker scripts stored in /lib on Hurd.
1574             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1575             cmdlist.add_command('strip', ['sh', '-c', (
1576                 'set -e; for f in $(%s); do '
1577                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1578                 'done' % (find_command, self.strip))])
1579         cmdlist.add_command('check', ['make', 'check'])
1580         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1581
1582 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1583     """Build policy for glibc during update-syscalls."""
1584
1585     def __init__(self, glibc):
1586         super().__init__(glibc)
1587         self.builddir = glibc.ctx.component_builddir(
1588             'update-syscalls', glibc.name, 'glibc')
1589         self.linuxdir = glibc.ctx.component_builddir(
1590             'update-syscalls', glibc.name, 'linux')
1591         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1592             glibc, self.linuxdir)
1593         self.configure_args.insert(
1594             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1595         # self.installdir not set because installation is not supported
1596
1597 class Glibc(object):
1598     """A configuration for building glibc."""
1599
1600     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1601                  cfg=None, ccopts=None, cflags=None):
1602         """Initialize a Glibc object."""
1603         self.ctx = compiler.ctx
1604         self.compiler = compiler
1605         if arch is None:
1606             self.arch = compiler.arch
1607         else:
1608             self.arch = arch
1609         if os_name is None:
1610             self.os = compiler.os
1611         else:
1612             self.os = os_name
1613         self.variant = variant
1614         if variant is None:
1615             self.name = '%s-%s' % (self.arch, self.os)
1616         else:
1617             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1618         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1619         if cfg is None:
1620             self.cfg = []
1621         else:
1622             self.cfg = cfg
1623         # ccopts contain ABI options and are passed to configure as CC / CXX.
1624         self.ccopts = ccopts
1625         # cflags contain non-ABI options like -g or -O and are passed to
1626         # configure as CFLAGS / CXXFLAGS.
1627         self.cflags = cflags
1628
1629     def tool_name(self, tool):
1630         """Return the name of a cross-compilation tool."""
1631         ctool = '%s-%s' % (self.compiler.triplet, tool)
1632         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1633             ctool = '%s %s' % (ctool, self.ccopts)
1634         return ctool
1635
1636     def build(self):
1637         """Generate commands to build this glibc."""
1638         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1639         installdir = self.ctx.glibc_installdir(self.name)
1640         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1641         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1642         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1643         cmdlist.add_command('check-compilers',
1644                             ['test', '-f',
1645                              os.path.join(self.compiler.installdir, 'ok')])
1646         cmdlist.use_path(self.compiler.bindir)
1647         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1648         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1649                                       logsdir)
1650
1651     def build_glibc(self, cmdlist, policy):
1652         """Generate commands to build this glibc, either as part of a compiler
1653         build or with the bootstrapped compiler (and in the latter case, run
1654         tests as well)."""
1655         cmdlist.create_use_dir(policy.builddir)
1656         policy.configure(cmdlist)
1657         cmdlist.add_command('build', ['make'])
1658         cmdlist.add_command('install', ['make', 'install',
1659                                         'install_root=%s' % policy.installdir])
1660         # GCC uses paths such as lib/../lib64, so make sure lib
1661         # directories always exist.
1662         mkdir_cmd = ['mkdir', '-p',
1663                      os.path.join(policy.installdir, 'lib')]
1664         if policy.use_usr:
1665             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1666         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1667         policy.extra_commands(cmdlist)
1668         cmdlist.cleanup_dir()
1669
1670     def update_syscalls(self):
1671         if self.os == 'gnu':
1672             # Hurd does not have system call tables that need updating.
1673             return
1674
1675         policy = GlibcPolicyForUpdateSyscalls(self)
1676         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1677         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1678         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1679         cmdlist.add_command('check-compilers',
1680                             ['test', '-f',
1681                              os.path.join(self.compiler.installdir, 'ok')])
1682         cmdlist.use_path(self.compiler.bindir)
1683
1684         install_linux_headers(policy.linux_policy, cmdlist)
1685
1686         cmdlist.create_use_dir(policy.builddir)
1687         policy.configure(cmdlist)
1688         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1689         cmdlist.cleanup_dir()
1690         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1691                                       cmdlist, logsdir)
1692
1693 class Command(object):
1694     """A command run in the build process."""
1695
1696     def __init__(self, desc, num, dir, path, command, always_run=False):
1697         """Initialize a Command object."""
1698         self.dir = dir
1699         self.path = path
1700         self.desc = desc
1701         trans = str.maketrans({' ': '-'})
1702         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1703         self.command = command
1704         self.always_run = always_run
1705
1706     @staticmethod
1707     def shell_make_quote_string(s):
1708         """Given a string not containing a newline, quote it for use by the
1709         shell and make."""
1710         assert '\n' not in s
1711         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1712             return s
1713         strans = str.maketrans({"'": "'\\''"})
1714         s = "'%s'" % s.translate(strans)
1715         mtrans = str.maketrans({'$': '$$'})
1716         return s.translate(mtrans)
1717
1718     @staticmethod
1719     def shell_make_quote_list(l, translate_make):
1720         """Given a list of strings not containing newlines, quote them for use
1721         by the shell and make, returning a single string.  If translate_make
1722         is true and the first string is 'make', change it to $(MAKE)."""
1723         l = [Command.shell_make_quote_string(s) for s in l]
1724         if translate_make and l[0] == 'make':
1725             l[0] = '$(MAKE)'
1726         return ' '.join(l)
1727
1728     def shell_make_quote(self):
1729         """Return this command quoted for the shell and make."""
1730         return self.shell_make_quote_list(self.command, True)
1731
1732
1733 class CommandList(object):
1734     """A list of commands run in the build process."""
1735
1736     def __init__(self, desc, keep):
1737         """Initialize a CommandList object."""
1738         self.cmdlist = []
1739         self.dir = None
1740         self.path = None
1741         self.desc = [desc]
1742         self.keep = keep
1743
1744     def desc_txt(self, desc):
1745         """Return the description to use for a command."""
1746         return '%s %s' % (' '.join(self.desc), desc)
1747
1748     def use_dir(self, dir):
1749         """Set the default directory for subsequent commands."""
1750         self.dir = dir
1751
1752     def use_path(self, path):
1753         """Set a directory to be prepended to the PATH for subsequent
1754         commands."""
1755         self.path = path
1756
1757     def push_subdesc(self, subdesc):
1758         """Set the default subdescription for subsequent commands (e.g., the
1759         name of a component being built, within the series of commands
1760         building it)."""
1761         self.desc.append(subdesc)
1762
1763     def pop_subdesc(self):
1764         """Pop a subdescription from the list of descriptions."""
1765         self.desc.pop()
1766
1767     def create_use_dir(self, dir):
1768         """Remove and recreate a directory and use it for subsequent
1769         commands."""
1770         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1771         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1772         self.use_dir(dir)
1773
1774     def add_command_dir(self, desc, dir, command, always_run=False):
1775         """Add a command to run in a given directory."""
1776         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1777                       command, always_run)
1778         self.cmdlist.append(cmd)
1779
1780     def add_command(self, desc, command, always_run=False):
1781         """Add a command to run in the default directory."""
1782         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1783                       self.path, command, always_run)
1784         self.cmdlist.append(cmd)
1785
1786     def cleanup_dir(self, desc='cleanup', dir=None):
1787         """Clean up a build directory.  If no directory is specified, the
1788         default directory is cleaned up and ceases to be the default
1789         directory."""
1790         if dir is None:
1791             dir = self.dir
1792             self.use_dir(None)
1793         if self.keep != 'all':
1794             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1795                                  always_run=(self.keep == 'none'))
1796
1797     def makefile_commands(self, wrapper, logsdir):
1798         """Return the sequence of commands in the form of text for a Makefile.
1799         The given wrapper script takes arguments: base of logs for
1800         previous command, or empty; base of logs for this command;
1801         description; directory; PATH addition; the command itself."""
1802         # prev_base is the base of the name for logs of the previous
1803         # command that is not always-run (that is, a build command,
1804         # whose failure should stop subsequent build commands from
1805         # being run, as opposed to a cleanup command, which is run
1806         # even if previous commands failed).
1807         prev_base = ''
1808         cmds = []
1809         for c in self.cmdlist:
1810             ctxt = c.shell_make_quote()
1811             if prev_base and not c.always_run:
1812                 prev_log = os.path.join(logsdir, prev_base)
1813             else:
1814                 prev_log = ''
1815             this_log = os.path.join(logsdir, c.logbase)
1816             if not c.always_run:
1817                 prev_base = c.logbase
1818             if c.dir is None:
1819                 dir = ''
1820             else:
1821                 dir = c.dir
1822             if c.path is None:
1823                 path = ''
1824             else:
1825                 path = c.path
1826             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1827             prelim_txt = Command.shell_make_quote_list(prelims, False)
1828             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1829         return '\n'.join(cmds)
1830
1831     def status_logs(self, logsdir):
1832         """Return the list of log files with command status."""
1833         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1834                 for c in self.cmdlist]
1835
1836
1837 def get_parser():
1838     """Return an argument parser for this module."""
1839     parser = argparse.ArgumentParser(description=__doc__)
1840     parser.add_argument('-j', dest='parallelism',
1841                         help='Run this number of jobs in parallel',
1842                         type=int, default=os.cpu_count())
1843     parser.add_argument('--keep', dest='keep',
1844                         help='Whether to keep all build directories, '
1845                         'none or only those from failed builds',
1846                         default='none', choices=('none', 'all', 'failed'))
1847     parser.add_argument('--replace-sources', action='store_true',
1848                         help='Remove and replace source directories '
1849                         'with the wrong version of a component')
1850     parser.add_argument('--strip', action='store_true',
1851                         help='Strip installed glibc libraries')
1852     parser.add_argument('--full-gcc', action='store_true',
1853                         help='Build GCC with all languages and libsanitizer')
1854     parser.add_argument('--shallow', action='store_true',
1855                         help='Do not download Git history during checkout')
1856     parser.add_argument('topdir',
1857                         help='Toplevel working directory')
1858     parser.add_argument('action',
1859                         help='What to do',
1860                         choices=('checkout', 'bot-cycle', 'bot',
1861                                  'host-libraries', 'compilers', 'glibcs',
1862                                  'update-syscalls', 'list-compilers',
1863                                  'list-glibcs'))
1864     parser.add_argument('configs',
1865                         help='Versions to check out or configurations to build',
1866                         nargs='*')
1867     return parser
1868
1869
1870 def main(argv):
1871     """The main entry point."""
1872     parser = get_parser()
1873     opts = parser.parse_args(argv)
1874     topdir = os.path.abspath(opts.topdir)
1875     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1876                   opts.strip, opts.full_gcc, opts.action,
1877                   shallow=opts.shallow)
1878     ctx.run_builds(opts.action, opts.configs)
1879
1880
1881 if __name__ == '__main__':
1882     main(sys.argv[1:])