Projetos/rpi-buildroot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
f2be0bb278f5ab2bb052d4a2420c00cbe7f27ff2
rpi-buildroot/support/scripts/gen-bootlin-toolchains
Thomas Petazzoni f2be0bb278 support/scripts/gen-bootlin-toolchains: improve dependencies of ARM toolchains 
The current description of the Bootlin ARM toolchains allows them to
be selected for ARM big endian configurations, which obviously doesn't
work as these Bootlin ARM toolchains are little endian only.

We fix this by adding BR2_arm in the list of conditions for those
toolchains.

Will allow to fix:

  http://autobuild.buildroot.net/results/7befbb686bb972016ba4e742976dcdb3fed1be11/

Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>
2022-06-14 16:49:27 +02:00

555 lines
20 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
import os.path
import re
import requests
import textwrap
BASE_URL = "https://toolchains.bootlin.com/downloads/releases/toolchains"
AUTOGENERATED_COMMENT = """# This file was auto-generated by support/scripts/gen-bootlin-toolchains
# Do not edit
"""
# In the below dict:
# - 'conditions' indicate the cumulative conditions under which the
# toolchain will be made available. In several situations, a given
# toolchain is usable on several architectures variants (for
# example, an ARMv6 toolchain can be used on ARMv7)
# - 'test_options' indicate one specific configuration where the
# toolchain can be used. It is used to create the runtime test
# cases. If 'test_options' does not exist, the code assumes it can
# be made equal to 'conditions'
# - 'prefix' is the prefix of the cross-compilation toolchain tools
arches = {
'aarch64': {
'conditions': ['BR2_aarch64'],
'prefix': 'aarch64',
},
'aarch64be': {
'conditions': ['BR2_aarch64_be'],
'prefix': 'aarch64_be',
},
'arcle-750d': {
'conditions': ['BR2_arcle', 'BR2_arc750d'],
'prefix': 'arc',
},
'arcle-hs38': {
'conditions': ['BR2_arcle', 'BR2_archs38'],
'prefix': 'arc',
},
'armv5-eabi': {
'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV5', 'BR2_ARM_EABI'],
'test_options': ['BR2_arm', 'BR2_arm926t', 'BR2_ARM_EABI'],
'prefix': 'arm',
},
'armv6-eabihf': {
'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV6', 'BR2_ARM_EABIHF'],
'test_options': ['BR2_arm', 'BR2_arm1176jzf_s', 'BR2_ARM_EABIHF'],
'prefix': 'arm',
},
'armv7-eabihf': {
'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV7A', 'BR2_ARM_EABIHF'],
'test_options': ['BR2_arm', 'BR2_cortex_a8', 'BR2_ARM_EABIHF'],
'prefix': 'arm',
},
'armv7m': {
'conditions': ['BR2_arm', 'BR2_ARM_CPU_ARMV7M'],
'test_options': ['BR2_arm', 'BR2_cortex_m4'],
'prefix': 'arm',
},
'm68k-68xxx': {
'conditions': ['BR2_m68k_m68k'],
'test_options': ['BR2_m68k', 'BR2_m68k_68040'],
'prefix': 'm68k',
},
'm68k-coldfire': {
'conditions': ['BR2_m68k_cf'],
'test_options': ['BR2_m68k', 'BR2_m68k_cf5208'],
'prefix': 'm68k',
},
'microblazebe': {
'conditions': ['BR2_microblazebe'],
'prefix': 'microblaze',
},
'microblazeel': {
'conditions': ['BR2_microblazeel'],
'prefix': 'microblazeel',
},
'mips32': {
# Not sure it could be used by other mips32 variants?
'conditions': ['BR2_mips', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips',
},
'mips32el': {
# Not sure it could be used by other mips32el variants?
'conditions': ['BR2_mipsel', 'BR2_mips_32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mipsel',
},
'mips32r5el': {
'conditions': ['BR2_mipsel', 'BR2_mips_32r5', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mipsel',
},
'mips32r6el': {
'conditions': ['BR2_mipsel', 'BR2_mips_32r6', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mipsel',
},
'mips64': {
# Not sure it could be used by other mips64 variants?
'conditions': ['BR2_mips64', 'BR2_mips_64', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64',
},
'mips64-n32': {
# Not sure it could be used by other mips64 variants?
'conditions': ['BR2_mips64', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64',
},
'mips64el-n32': {
# Not sure it could be used by other mips64el variants?
'conditions': ['BR2_mips64el', 'BR2_mips_64', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64el',
},
'mips64r6el-n32': {
'conditions': ['BR2_mips64el', 'BR2_mips_64r6', 'BR2_MIPS_NABI32', '!BR2_MIPS_SOFT_FLOAT'],
'prefix': 'mips64el',
},
'nios2': {
'conditions': ['BR2_nios2'],
'prefix': 'nios2',
},
'openrisc': {
'conditions': ['BR2_or1k'],
'prefix': 'or1k',
},
'powerpc-440fp': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'],
'prefix': 'powerpc',
},
'powerpc-e300c3': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'],
'prefix': 'powerpc',
},
'powerpc-e500mc': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
'prefix': 'powerpc',
},
'powerpc64-e5500': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
'prefix': 'powerpc64',
},
'powerpc64-e6500': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'],
'prefix': 'powerpc64',
},
'powerpc64-power8': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
'prefix': 'powerpc64',
},
'powerpc64le-power8': {
'conditions': ['BR2_powerpc64le', 'BR2_powerpc_power8'],
'prefix': 'powerpc64le',
},
'riscv32-ilp32d': {
'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_32', 'BR2_RISCV_ABI_ILP32D'],
'prefix': 'riscv32',
},
'riscv64-lp64d': {
'conditions': ['BR2_riscv', 'BR2_riscv_g', 'BR2_RISCV_64', 'BR2_RISCV_ABI_LP64D', 'BR2_USE_MMU'],
'prefix': 'riscv64',
},
's390x-z13': {
'conditions': ['BR2_s390x', 'BR2_s390x_z13'],
'prefix': 's390x',
},
'sh-sh4': {
'conditions': ['BR2_sh', 'BR2_sh4'],
'prefix': 'sh4',
},
'sh-sh4aeb': {
'conditions': ['BR2_sh', 'BR2_sh4aeb'],
'prefix': 'sh4aeb',
},
'sparc64': {
'conditions': ['BR2_sparc64', 'BR2_sparc_v9'],
'prefix': 'sparc64',
},
'sparcv8': {
'conditions': ['BR2_sparc', 'BR2_sparc_v8'],
'prefix': 'sparc',
},
'x86-64': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64'],
'prefix': 'x86_64',
},
'x86-64-v2': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v2'],
'prefix': 'x86_64',
},
'x86-64-v3': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42',
'BR2_X86_CPU_HAS_AVX',
'BR2_X86_CPU_HAS_AVX2'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v3'],
'prefix': 'x86_64',
},
'x86-64-v4': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42',
'BR2_X86_CPU_HAS_AVX',
'BR2_X86_CPU_HAS_AVX2',
'BR2_X86_CPU_HAS_AVX512'],
'test_options': ['BR2_x86_64', 'BR2_x86_x86_64_v4'],
'prefix': 'x86_64',
},
'x86-64-core-i7': {
'conditions': ['BR2_x86_64',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3',
'BR2_X86_CPU_HAS_SSE4',
'BR2_X86_CPU_HAS_SSE42'],
'test_options': ['BR2_x86_64', 'BR2_x86_corei7'],
'prefix': 'x86_64',
},
'x86-core2': {
'conditions': ['BR2_i386',
'BR2_X86_CPU_HAS_MMX',
'BR2_X86_CPU_HAS_SSE',
'BR2_X86_CPU_HAS_SSE2',
'BR2_X86_CPU_HAS_SSE3',
'BR2_X86_CPU_HAS_SSSE3'],
'test_options': ['BR2_i386', 'BR2_x86_core2'],
'prefix': 'i686',
},
'x86-i686': {
'conditions': ['BR2_i386',
'!BR2_x86_i486',
'!BR2_x86_i586',
'!BR2_x86_x1000'],
'test_options': ['BR2_i386',
'BR2_x86_i686'],
'prefix': 'i686',
},
'xtensa-lx60': {
'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'],
'prefix': 'xtensa',
},
}
class Toolchain:
def __init__(self, arch, libc, variant, version):
self.arch = arch
self.libc = libc
self.variant = variant
self.version = version
self.fname_prefix = "%s--%s--%s-%s" % (self.arch, self.libc, self.variant, self.version)
self.option_name = "BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_%s_%s_%s" % \
(self.arch.replace("-", "_").upper(), self.libc.upper(), self.variant.replace("-", "_").upper())
self.fragment = requests.get(self.fragment_url).text.split("\n")
self.sha256 = requests.get(self.hash_url).text.split(" ")[0]
@property
def tarball_url(self):
return os.path.join(BASE_URL, self.arch, "tarballs",
self.fname_prefix + ".tar.bz2")
@property
def hash_url(self):
return os.path.join(BASE_URL, self.arch, "tarballs",
self.fname_prefix + ".sha256")
@property
def fragment_url(self):
return os.path.join(BASE_URL, self.arch, "fragments",
self.fname_prefix + ".frag")
def gen_config_in_options(self, f):
f.write("config %s\n" % self.option_name)
f.write("\tbool \"%s %s %s %s\"\n" %
(self.arch, self.libc, self.variant, self.version))
depends = []
selects = []
for c in arches[self.arch]['conditions']:
depends.append(c)
for frag in self.fragment:
# libc type
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
# glibc needs mmu support
if "BR2_USE_MMU" not in depends:
depends.append("BR2_USE_MMU")
# glibc doesn't support static only configuration
depends.append("!BR2_STATIC_LIBS")
selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
# musl needs mmu support
if "BR2_USE_MMU" not in depends:
depends.append("BR2_USE_MMU")
selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
# gcc version
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_GCC_"):
m = re.match("^BR2_TOOLCHAIN_EXTERNAL_GCC_([0-9_]*)=y$", frag)
assert m, "Cannot get gcc version for toolchain %s" % self.fname_prefix
selects.append("BR2_TOOLCHAIN_GCC_AT_LEAST_%s" % m[1])
# kernel headers version
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HEADERS_"):
m = re.match("^BR2_TOOLCHAIN_EXTERNAL_HEADERS_([0-9_]*)=y$", frag)
assert m, "Cannot get kernel headers version for toolchain %s" % self.fname_prefix
selects.append("BR2_TOOLCHAIN_HEADERS_AT_LEAST_%s" % m[1])
# C++
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CXX"):
selects.append("BR2_INSTALL_LIBSTDCPP")
# SSP
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_SSP"):
selects.append("BR2_TOOLCHAIN_HAS_SSP")
# wchar
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_WCHAR"):
selects.append("BR2_USE_WCHAR")
# locale
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_LOCALE"):
# locale implies the availability of wchar
selects.append("BR2_USE_WCHAR")
selects.append("BR2_ENABLE_LOCALE")
# thread support
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS"):
selects.append("BR2_TOOLCHAIN_HAS_THREADS")
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_DEBUG"):
selects.append("BR2_TOOLCHAIN_HAS_THREADS_DEBUG")
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_HAS_THREADS_NPTL"):
selects.append("BR2_TOOLCHAIN_HAS_THREADS_NPTL")
# RPC
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_INET_RPC"):
selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
# D language
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_DLANG"):
selects.append("BR2_TOOLCHAIN_HAS_DLANG")
# fortran
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_FORTRAN"):
selects.append("BR2_TOOLCHAIN_HAS_FORTRAN")
# OpenMP
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
for depend in depends:
f.write("\tdepends on %s\n" % depend)
for select in selects:
f.write("\tselect %s\n" % select)
f.write("\thelp\n")
desc = "Bootlin toolchain for the %s architecture, using the %s C library. " % \
(self.arch, self.libc)
if self.variant == "stable":
desc += "This is a stable version, which means it is using stable and proven versions of gcc, gdb and binutils."
else:
desc += "This is a bleeding-edge version, which means it is using the latest versions of gcc, gdb and binutils."
f.write(textwrap.fill(desc, width=62, initial_indent="\t ", subsequent_indent="\t ") + "\n")
f.write("\n")
f.write("\t https://toolchains.bootlin.com/\n")
f.write("\n")
def gen_mk(self, f):
f.write("ifeq ($(%s),y)\n" % self.option_name)
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION = %s\n" % self.version)
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SOURCE = %s--%s--%s-$(TOOLCHAIN_EXTERNAL_BOOTLIN_VERSION).tar.bz2\n" %
(self.arch, self.libc, self.variant))
f.write("TOOLCHAIN_EXTERNAL_BOOTLIN_SITE = %s\n" %
os.path.join(BASE_URL, self.arch, "tarballs"))
f.write("endif\n\n")
pass
def gen_hash(self, f):
f.write("# From %s\n" % self.hash_url)
f.write("sha256 %s %s\n" % (self.sha256, os.path.basename(self.tarball_url)))
def gen_test(self, f):
if self.variant == "stable":
variant = "Stable"
else:
variant = "BleedingEdge"
testname = "TestExternalToolchainBootlin" + \
self.arch.replace("-", "").capitalize() + \
self.libc.capitalize() + variant
f.write("\n\n")
f.write("class %s(TestExternalToolchain):\n" % testname)
f.write(" config = \"\"\"\n")
if 'test_options' in arches[self.arch]:
test_options = arches[self.arch]['test_options']
else:
test_options = arches[self.arch]['conditions']
for opt in test_options:
if opt.startswith("!"):
f.write(" # %s is not set\n" % opt[1:])
else:
f.write(" %s=y\n" % opt)
f.write(" BR2_TOOLCHAIN_EXTERNAL=y\n")
f.write(" BR2_TOOLCHAIN_EXTERNAL_BOOTLIN=y\n")
f.write(" %s=y\n" % self.option_name)
f.write(" # BR2_TARGET_ROOTFS_TAR is not set\n")
f.write(" \"\"\"\n")
f.write(" toolchain_prefix = \"%s-linux\"\n" % arches[self.arch]['prefix'])
f.write("\n")
f.write(" def test_run(self):\n")
f.write(" TestExternalToolchain.common_check(self)\n")
def __repr__(self):
return "Toolchain(arch=%s libc=%s variant=%s version=%s, option=%s)" % \
(self.arch, self.libc, self.variant, self.version, self.option_name)
def get_toolchains():
toolchains = list()
for arch, details in arches.items():
print(arch)
url = os.path.join(BASE_URL, arch, "available_toolchains")
page = requests.get(url).text
fnames = sorted(re.findall(r'<td><a href="(\w[^"]+)"', page))
# This dict will allow us to keep only the latest version for
# each toolchain.
tmp = dict()
for fname in fnames:
parts = fname.split('--')
assert parts[0] == arch, "Arch does not match: %s vs. %s" % (parts[0], arch)
libc = parts[1]
if parts[2].startswith("stable-"):
variant = "stable"
version = parts[2][len("stable-"):]
elif parts[2].startswith("bleeding-edge-"):
variant = "bleeding-edge"
version = parts[2][len("bleeding-edge-"):]
tmp[(arch, libc, variant)] = version
toolchains += [Toolchain(k[0], k[1], k[2], v) for k, v in tmp.items()]
return toolchains
def gen_config_in_options(toolchains, fpath):
with open(fpath, "w") as f:
f.write(AUTOGENERATED_COMMENT)
f.write("config BR2_TOOLCHAIN_EXTERNAL_BOOTLIN_ARCH_SUPPORTS\n")
f.write("\tbool\n")
for arch, details in arches.items():
conditions = details['conditions'].copy()
if set([t.libc for t in toolchains if t.arch == arch]) == set(['glibc']):
conditions.append("!BR2_STATIC_LIBS")
f.write("\tdefault y if %s\n" % " && ".join(conditions))
f.write("\n")
f.write("if BR2_TOOLCHAIN_EXTERNAL_BOOTLIN\n\n")
f.write("config BR2_TOOLCHAIN_EXTERNAL_PREFIX\n")
f.write("\tdefault \"$(ARCH)-linux\"\n")
f.write("\n")
f.write("config BR2_PACKAGE_PROVIDES_TOOLCHAIN_EXTERNAL\n")
f.write("\tdefault \"toolchain-external-bootlin\"\n")
f.write("\n")
f.write("choice\n")
f.write("\tprompt \"Bootlin toolchain variant\"\n")
for toolchain in toolchains:
toolchain.gen_config_in_options(f)
f.write("endchoice\n")
f.write("endif\n")
def gen_mk(toolchains, fpath):
with open(fpath, "w") as f:
f.write("#" * 80 + "\n")
f.write("#\n")
f.write("# toolchain-external-bootlin\n")
f.write("#\n")
f.write("#" * 80 + "\n")
f.write("\n")
f.write(AUTOGENERATED_COMMENT)
for toolchain in toolchains:
toolchain.gen_mk(f)
f.write("$(eval $(toolchain-external-package))\n")
def gen_hash(toolchains, fpath):
with open(fpath, "w") as f:
f.write(AUTOGENERATED_COMMENT)
for toolchain in toolchains:
toolchain.gen_hash(f)
def gen_runtime_test(toolchains, fpath):
with open(fpath, "w") as f:
f.write(AUTOGENERATED_COMMENT)
f.write("from tests.toolchain.test_external import TestExternalToolchain\n")
for toolchain in toolchains:
toolchain.gen_test(f)
def gen_toolchains(toolchains):
maindir = "toolchain/toolchain-external/toolchain-external-bootlin"
gen_config_in_options(toolchains, os.path.join(maindir, "Config.in.options"))
gen_mk(toolchains, os.path.join(maindir, "toolchain-external-bootlin.mk"))
gen_hash(toolchains, os.path.join(maindir, "toolchain-external-bootlin.hash"))
gen_runtime_test(toolchains,
os.path.join("support", "testing", "tests", "toolchain", "test_external_bootlin.py"))
toolchains = get_toolchains()
gen_toolchains(toolchains)
Reference in New Issue View Git Blame Copy Permalink