buildroot/support/scripts/gen-bootlin-toolchains

#!/usr/bin/env python3

import os.path
import re
import requests
import textwrap
import sys

BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"

AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
# Do not edit
"""

# In the below dict:

# - 'conditions' indicate the cumulative conditions under which the
#   toolchain will be made available. In several situations, a given
#   toolchain is usable on several architectures variants (for
#   example, an ARMv6 toolchain can be used on ARMv7)
# - 'test_options' indicate one specific configuration where the
#   toolchain can be used. It is used to create the runtime test
#   cases. If 'test_options' does not exist, the code assumes it can
#   be made equal to 'conditions'
# - 'prefix' is the prefix of the cross-compilation toolchain tools

arches = {
    'aarch64': {
        'conditions': ['BR2_aarch64'],
        'prefix': 'aarch64',
    },
    'aarch64be': {
        'conditions': ['BR2_aarch64_be'],
        'prefix': 'aarch64_be',
    },
    'arcle-750d': {
        'conditions': ['BR2_arcle', 'BR2_arc750d'],
        'prefix': 'arc',
    },
    'arcle-hs38': {
        'conditions': ['BR2_arcle', 'BR2_archs38'],
        'prefix': 'arc',
    },
    'armv5-eabi': {
        'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
        'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
        'prefix': 'arm',
    },
    'armv6-eabihf': {
        'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
        'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
        'prefix': 'arm',
    },
    'armv7-eabihf': {
        'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
        'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
        'prefix': 'arm',
    },
    'armebv7-eabihf': {
        'conditions': ['BR2_armeb', 'BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
        'test_options': ['BR2_armeb', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
        'prefix': 'armeb',
    },
    'armv7m': {
        'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV7M'],
        'test_options': ['BR2_arm', 'BR2_cortex_m4'],
        'prefix': 'arm',
    },
    'm68k-68xxx': {
        'conditions': ['BR2_m68k_m68k'],
        'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
        'prefix': 'm68k',
    },
    'm68k-coldfire': {
        'conditions': ['BR2_m68k_cf'],
        'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
        'prefix': 'm68k',
    },
    'microblazebe': {
        'conditions': ['BR2_microblazebe'],
        'prefix': 'microblaze',
        'gdbserver': False
    },
    'microblazeel': {
        'conditions': ['BR2_microblazeel'],
        'prefix': 'microblazeel',
        'gdbserver': False
    },
    'mips32': {
        # Not sure it could be used by other mips32 variants?
        'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mips',
    },
    'mips32el': {
        # Not sure it could be used by other mips32el variants?
        'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mipsel',
    },
    'mips32r5el': {
        'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mipsel',
    },
    'mips32r6el': {
        'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mipsel',
    },
    'mips64-n32': {
        # Not sure it could be used by other mips64 variants?
        'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mips64',
    },
    'mips64el-n32': {
        # Not sure it could be used by other mips64el variants?
        'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mips64el',
    },
    'mips64r6el-n32': {
        'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
        'prefix': 'mips64el',
    },
    'nios2': {
        'conditions': ['BR2_nios2'],
        'prefix': 'nios2',
    },
    'openrisc': {
        'conditions': ['BR2_or1k'],
        'prefix': 'or1k',
        'gdbserver': False,
    },
    'powerpc-440fp': {
        # Not sure it could be used by other powerpc variants?
        'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'],
        'prefix': 'powerpc',
    },
    'powerpc-e300c3': {
        # Not sure it could be used by other powerpc variants?
        'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'],
        'prefix': 'powerpc',
    },
    'powerpc-e500mc': {
        # Not sure it could be used by other powerpc variants?
        'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
        'prefix': 'powerpc',
    },
    'powerpc64-e5500': {
        'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
        'prefix': 'powerpc64',
    },
    'powerpc64-e6500': {
        'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'],
        'prefix': 'powerpc64',
    },
    'powerpc64-power8': {
        'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
        'prefix': 'powerpc64',
    },
    'powerpc64le-power8': {
        'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
        'prefix': 'powerpc64le',
    },
    'riscv32-ilp32d': {
        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
        'prefix': 'riscv32',
    },
    'riscv64-lp64d': {
        'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64D', 'BR2_USE_MMU'],
        'prefix': 'riscv64',
    },
    's390x-z13': {
        'conditions': ['BR2_s390x', 'BR2_s390x_z13'],
        'prefix': 's390x',
    },
    'sh-sh4': {
        'conditions': ['BR2_sh', 'BR2_sh4'],
        'prefix': 'sh4',
    },
    'sh-sh4aeb': {
        'conditions': ['BR2_sh', 'BR2_sh4aeb'],
        'prefix': 'sh4aeb',
    },
    'sparc64': {
        'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
        'prefix': 'sparc64',
    },
    'sparcv8': {
        'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
        'prefix': 'sparc',
    },
    'x86-64': {
        'conditions': ['BR2_x86_64',
                       'BR2_X86_CPU_HAS_MMX',
                       'BR2_X86_CPU_HAS_SSE',
                       'BR2_X86_CPU_HAS_SSE2'],
        'test_options': ['BR2_x86_64', 'BR2_x86_x86_64'],
        'prefix': 'x86_64',
    },
    'x86-64-v2': {
        'conditions': ['BR2_x86_64',
                       'BR2_X86_CPU_HAS_MMX',
                       'BR2_X86_CPU_HAS_SSE',
                       'BR2_X86_CPU_HAS_SSE2',
                       'BR2_X86_CPU_HAS_SSE3',
                       'BR2_X86_CPU_HAS_SSSE3',
                       'BR2_X86_CPU_HAS_SSE4',
                       'BR2_X86_CPU_HAS_SSE42'],
        'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v2'],
        'prefix': 'x86_64',
    },
    'x86-64-v3': {
        'conditions': ['BR2_x86_64',
                       'BR2_X86_CPU_HAS_MMX',
                       'BR2_X86_CPU_HAS_SSE',
                       'BR2_X86_CPU_HAS_SSE2',
                       'BR2_X86_CPU_HAS_SSE3',
                       'BR2_X86_CPU_HAS_SSSE3',
                       'BR2_X86_CPU_HAS_SSE4',
                       'BR2_X86_CPU_HAS_SSE42',
                       'BR2_X86_CPU_HAS_AVX',
                       'BR2_X86_CPU_HAS_AVX2'],
        'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v3'],
        'prefix': 'x86_64',
    },
    'x86-64-v4': {
        'conditions': ['BR2_x86_64',
                       'BR2_X86_CPU_HAS_MMX',
                       'BR2_X86_CPU_HAS_SSE',
                       'BR2_X86_CPU_HAS_SSE2',
                       'BR2_X86_CPU_HAS_SSE3',
                       'BR2_X86_CPU_HAS_SSSE3',
                       'BR2_X86_CPU_HAS_SSE4',
                       'BR2_X86_CPU_HAS_SSE42',
                       'BR2_X86_CPU_HAS_AVX',
                       'BR2_X86_CPU_HAS_AVX2',
                       'BR2_X86_CPU_HAS_AVX512'],
        'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v4'],
        'prefix': 'x86_64',
    },
    'x86-64-core-i7': {
        'conditions': ['BR2_x86_64',
                       'BR2_X86_CPU_HAS_MMX',
                       'BR2_X86_CPU_HAS_SSE',
                       'BR2_X86_CPU_HAS_SSE2',
                       'BR2_X86_CPU_HAS_SSE3',
                       'BR2_X86_CPU_HAS_SSSE3',
                       'BR2_X86_CPU_HAS_SSE4',
                       'BR2_X86_CPU_HAS_SSE42'],
        'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
        'prefix': 'x86_64',
    },
    'x86-core2': {
        'conditions': ['BR2_i386',
                       'BR2_X86_CPU_HAS_MMX',
                       'BR2_X86_CPU_HAS_SSE',
                       'BR2_X86_CPU_HAS_SSE2',
                       'BR2_X86_CPU_HAS_SSE3',
                       'BR2_X86_CPU_HAS_SSSE3'],
        'test_options': ['BR2_i386', 'BR2_x86_core2'],
        'prefix': 'i686',
    },
    'x86-i686': {
        'conditions': ['BR2_i386',
                       '!BR2_x86_i486',
                       '!BR2_x86_i586',
                       '!BR2_x86_x1000',
                       '!BR2_x86_pentium_mmx',
                       '!BR2_x86_geode',
                       '!BR2_x86_c3',
                       '!BR2_x86_winchip_c6',
                       '!BR2_x86_winchip2'],
        'test_options': ['BR2_i386',
                         'BR2_x86_i686'],
        'prefix': 'i686',
    },
    'xtensa-lx60': {
        'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'],
        'prefix': 'xtensa',
    },
}


class Toolchain:
    def __init__(self, arch, libc, variant, version):
        self.arch = arch
        self.libc = libc
        self.variant = variant
        self.version = version
        self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
        self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
            (self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
        self.fragment = requests.get(self.fragment_url).text.split("\n")
        self.sha256 = requests.get(self.hash_url).text.split(" ")[0]

    @property
    def tarball_url(self):
        return os.path.join(BASE_URL, self.arch, "tarballs",
                            self.fname_prefix + ".tar.bz2")

    @property
    def hash_url(self):
        return os.path.join(BASE_URL, self.arch, "tarballs",
                            self.fname_prefix + ".sha256")

    @property
    def fragment_url(self):
        return os.path.join(BASE_URL, self.arch, "fragments",
                            self.fname_prefix + ".frag")

    def gen_config_in_options(self, f):
        f.write("config %s\n" % self.option_name)
        f.write("\tbool \"%s %s %s %s\"\n" %
                (self.arch, self.libc, self.variant, self.version))
        depends = []
        selects = []

        for c in arches[self.arch]['conditions']:
            depends.append(c)

        if not arches[self.arch].get('gdbserver', True):
            selects.append("BR2_TOOLCHAIN_EXTERNAL_HAS_NO_GDBSERVER")

        for frag in self.fragment:
            # libc type
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
                selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
                # glibc needs mmu support
                if "BR2_USE_MMU" not in depends:
                    depends.append("BR2_USE_MMU")
                # glibc doesn't support static only configuration
                depends.append("!BR2_STATIC_LIBS")
                selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
            elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
                # musl needs mmu support
                if "BR2_USE_MMU" not in depends:
                    depends.append("BR2_USE_MMU")
                selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")

            # gcc version
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
                assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
                selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])

            # kernel headers version
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
                m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
                assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
                selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])

            # C++
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
                selects.append("BR2_INSTALL_LIBSTDCPP")

            # SSP
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
                selects.append("BR2_TOOLCHAIN_HAS_SSP")

            # wchar
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
                selects.append("BR2_USE_WCHAR")

            # locale
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
                # locale implies the availability of wchar
                selects.append("BR2_USE_WCHAR")
                selects.append("BR2_ENABLE_LOCALE")

            # thread support
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
                selects.append("BR2_TOOLCHAIN_HAS_THREADS")

            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
                selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")

            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
                selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")

            # RPC
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
                selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")

            # D language
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
                selects.append("BR2_TOOLCHAIN_HAS_DLANG")

            # fortran
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
                selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")

            # OpenMP
            if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
                selects.append("BR2_TOOLCHAIN_HAS_OPENMP")

        for depend in depends:
            f.write("\tdepends on %s\n" % depend)

        for select in selects:
            f.write("\tselect %s\n" % select)

        f.write("\thelp\n")

        desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
            (self.arch, self.libc)

        if self.variant == "stable":
            desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
        else:
            desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."

        f.write(textwrap.fill(desc, width=62, initial_indent="\t  ", subsequent_indent="\t  ") + "\n")
        f.write("\n")
        f.write("\t  https://toolchains.bootlin.com/\n")

        f.write("\n")

    def gen_mk(self, f):
        f.write("ifeq ($(%s),y)\n" % self.option_name)
        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
                (self.arch, self.libc, self.variant))
        f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
                os.path.join(BASE_URL, self.arch, "tarballs"))
        f.write("endif\n\n")
        pass

    def gen_hash(self, f):
        f.write("# From %s\n" % self.hash_url)
        f.write("sha256  %s  %s\n" % (self.sha256, os.path.basename(self.tarball_url)))

    def gen_test(self, f):
        if self.variant == "stable":
            variant = "Stable"
        else:
            variant = "BleedingEdge"
        testname = "TestExternalToolchainBootlin" + \
            self.arch.replace("-", "").capitalize() + \
            self.libc.capitalize() + variant
        f.write("\n\n")
        f.write("class %s(TestExternalToolchain):\n" % testname)
        f.write("    config = \"\"\"\n")
        if 'test_options' in arches[self.arch]:
            test_options = arches[self.arch]['test_options']
        else:
            test_options = arches[self.arch]['conditions']
        for opt in test_options:
            if opt.startswith("!"):
                f.write("        # %s is not set\n" % opt[1:])
            else:
                f.write("        %s=y\n" % opt)
        f.write("        BR2_TOOLCHAIN_EXTERNAL=y\n")
        f.write("        BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
        f.write("        %s=y\n" % self.option_name)
        f.write("        # BR2_TARGET_ROOTFS_TAR is not set\n")
        f.write("        \"\"\"\n")
        f.write("    toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
        f.write("\n")
        f.write("    def test_run(self):\n")
        f.write("        TestExternalToolchain.common_check(self)\n")

    def __repr__(self):
        return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
            (self.arch, self.libc, self.variant, self.version, self.option_name)


def get_toolchains():
    toolchains = list()
    for arch, details in arches.items():
        print(arch)
        url = os.path.join(BASE_URL, arch, "available_toolchains")
        page = requests.get(url).text
        fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
        # This dict will allow us to keep only the latest version for
        # each toolchain.
        tmp = dict()
        for fname in fnames:
            parts = fname.split('--')
            assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
            libc = parts[1]
            if parts[2].startswith("stable-"):
                variant = "stable"
                version = parts[2][len("stable-"):]
            elif parts[2].startswith("bleeding-edge-"):
                variant = "bleeding-edge"
                version = parts[2][len("bleeding-edge-"):]
            tmp[(arch, libc, variant)] = version

        if len(tmp) == 0:
            print("ERROR: no toolchains found for architecture %s" % arch)
            sys.exit(1)

        toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]

    return toolchains


def gen_config_in_options(toolchains, fpath):
    with open(fpath, "w") as f:
        f.write(AUTOGENERATED_COMMENT)

        f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
        f.write("\tbool\n")
        for arch, details in arches.items():
            conditions = details['conditions'].copy()
            if set([t.libc for t in toolchains if t.arch == arch]) == set(['glibc']):
                conditions.append("!BR2_STATIC_LIBS")
            f.write("\tdefault y if %s\n" % " && ".join(conditions))
        f.write("\n")

        f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")

        f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
        f.write("\tdefault \"$(ARCH)-linux\"\n")

        f.write("\n")

        f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
        f.write("\tdefault \"toolchain-external-bootlin\"\n")

        f.write("\n")

        f.write("choice\n")
        f.write("\tprompt \"Bootlin toolchain variant\"\n")

        for toolchain in toolchains:
            toolchain.gen_config_in_options(f)

        f.write("endchoice\n")
        f.write("endif\n")


def gen_mk(toolchains, fpath):
    with open(fpath, "w") as f:
        f.write("#" * 80 + "\n")
        f.write("#\n")
        f.write("# toolchain-external-bootlin\n")
        f.write("#\n")
        f.write("#" * 80 + "\n")
        f.write("\n")
        f.write(AUTOGENERATED_COMMENT)
        for toolchain in toolchains:
            toolchain.gen_mk(f)
        f.write("$(eval $(toolchain-external-package))\n")


def gen_hash(toolchains, fpath):
    with open(fpath, "w") as f:
        f.write(AUTOGENERATED_COMMENT)
        for toolchain in toolchains:
            toolchain.gen_hash(f)


def gen_runtime_test(toolchains, fpath):
    with open(fpath, "w") as f:
        f.write(AUTOGENERATED_COMMENT)
        f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
        for toolchain in toolchains:
            toolchain.gen_test(f)


def gen_toolchains(toolchains):
    maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
    gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
    gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
    gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
    gen_runtime_test(toolchains,
                     os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))


toolchains = get_toolchains()
gen_toolchains(toolchains)