aboutsummaryrefslogtreecommitdiff
path: root/support/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'support/scripts')
-rwxr-xr-xsupport/scripts/apply-patches.sh9
-rwxr-xr-xsupport/scripts/boot-qemu-image.py10
-rwxr-xr-xsupport/scripts/br2-external18
-rw-r--r--support/scripts/cpedb.py174
-rwxr-xr-xsupport/scripts/cve-checker196
-rwxr-xr-xsupport/scripts/cve.py50
-rwxr-xr-xsupport/scripts/gen-bootlin-toolchains34
-rwxr-xr-xsupport/scripts/generate-gitlab-ci-yml141
-rwxr-xr-xsupport/scripts/pkg-stats219
-rwxr-xr-xsupport/scripts/setlocalversion16
10 files changed, 527 insertions, 340 deletions
diff --git a/support/scripts/apply-patches.sh b/support/scripts/apply-patches.sh
index 66fef262ee..9fb488c570 100755
--- a/support/scripts/apply-patches.sh
+++ b/support/scripts/apply-patches.sh
@@ -113,13 +113,13 @@ function apply_patch {
echo " to be applied : ${path}/${patch}"
exit 1
fi
- if grep -q "^rename from" ${path}/${patch} && \
- grep -q "^rename to" ${path}/${patch} ; then
+ if ${uncomp} "${path}/$patch" | grep -q "^rename from" && \
+ ${uncomp} "${path}/$patch" | grep -q "^rename to" ; then
echo "Error: patch contains some renames, not supported by old patch versions"
exit 1
fi
echo "${path}/${patch}" >> ${builddir}/.applied_patches_list
- ${uncomp} "${path}/$patch" | patch -g0 -p1 -E -d "${builddir}" -t -N $silent
+ ${uncomp} "${path}/$patch" | patch -g0 -p1 -E --no-backup-if-mismatch -d "${builddir}" -t -N $silent
if [ $? != 0 ] ; then
echo "Patch failed! Please fix ${patch}!"
exit 1
@@ -168,6 +168,3 @@ if [ "`find $builddir/ '(' -name '*.rej' -o -name '.*.rej' ')' -print`" ] ; then
echo "Aborting. Reject files found."
exit 1
fi
-
-# Remove backup files
-find $builddir/ '(' -name '*.orig' -o -name '.*.orig' ')' -exec rm -f {} \;
diff --git a/support/scripts/boot-qemu-image.py b/support/scripts/boot-qemu-image.py
index dbbba552ad..0d4ad825fc 100755
--- a/support/scripts/boot-qemu-image.py
+++ b/support/scripts/boot-qemu-image.py
@@ -18,6 +18,10 @@ def main():
if not sys.argv[1].startswith('qemu_'):
sys.exit(0)
+ if not os.path.exists('output/images/start-qemu.sh'):
+ print('qemu-start.sh is missing, cannot test.')
+ sys.exit(0)
+
qemu_start = os.path.join(os.getcwd(), 'output/images/start-qemu.sh')
child = pexpect.spawn(qemu_start, ['serial-only'],
@@ -32,7 +36,7 @@ def main():
time.sleep(1)
try:
- child.expect(["buildroot login:", pexpect.TIMEOUT], timeout=60)
+ child.expect(["buildroot login:"], timeout=60)
except pexpect.EOF as e:
# Some emulations require a fork of qemu-system, which may be
# missing on the system, and is not provided by Buildroot.
@@ -54,7 +58,7 @@ def main():
child.sendline("root\r")
try:
- child.expect(["# ", pexpect.TIMEOUT], timeout=60)
+ child.expect(["# "], timeout=60)
except pexpect.EOF:
print("Cannot connect to shell")
sys.exit(1)
@@ -65,7 +69,7 @@ def main():
child.sendline("poweroff\r")
try:
- child.expect(["System halted", pexpect.TIMEOUT], timeout=60)
+ child.expect(["System halted"], timeout=60)
child.expect(pexpect.EOF)
except pexpect.EOF:
pass
diff --git a/support/scripts/br2-external b/support/scripts/br2-external
index ededd2d900..56dc6f53ca 100755
--- a/support/scripts/br2-external
+++ b/support/scripts/br2-external
@@ -161,6 +161,8 @@ do_kconfig() {
toolchains
jpeg
openssl
+ skeleton
+ init
)
for br2 in "${items[@]}"; do
@@ -224,6 +226,22 @@ do_kconfig() {
else
printf '# No openssl from: %s\n\n' "${br2_desc}"
fi >>"${outputdir}/.br2-external.in.openssl"
+
+ if [ -f "${br2_ext}/provides/skeleton.in" ]; then
+ printf 'comment "skeleton from: %s"\n' "${br2_desc}"
+ printf 'source "%s/provides/skeleton.in"\n' "${br2_ext}"
+ printf '\n'
+ else
+ printf '# No skeleton from: %s\n\n' "${br2_desc}"
+ fi >>"${outputdir}/.br2-external.in.skeleton"
+
+ if [ -f "${br2_ext}/provides/init.in" ]; then
+ printf 'comment "init from: %s"\n' "${br2_desc}"
+ printf 'source "%s/provides/init.in"\n' "${br2_ext}"
+ printf '\n'
+ else
+ printf '# No init from: %s\n\n' "${br2_desc}"
+ fi >>"${outputdir}/.br2-external.in.init"
done
printf 'endmenu\n' >>"${outputdir}/.br2-external.in.menus"
diff --git a/support/scripts/cpedb.py b/support/scripts/cpedb.py
new file mode 100644
index 0000000000..f4daf56124
--- /dev/null
+++ b/support/scripts/cpedb.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+
+import xml.etree.ElementTree as ET
+from xml.etree.ElementTree import Element, SubElement
+import gzip
+import os
+import requests
+import time
+from xml.dom import minidom
+
+VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
+
+CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
+
+ns = {
+ '': 'http://cpe.mitre.org/dictionary/2.0',
+ 'cpe-23': 'http://scap.nist.gov/schema/cpe-extension/2.3',
+ 'xml': 'http://www.w3.org/XML/1998/namespace'
+}
+
+
+class CPE:
+ def __init__(self, cpe_str, titles, refs):
+ self.cpe_str = cpe_str
+ self.titles = titles
+ self.references = refs
+ self.cpe_cur_ver = "".join(self.cpe_str.split(":")[5:6])
+
+ def update_xml_dict(self):
+ ET.register_namespace('', 'http://cpe.mitre.org/dictionary/2.0')
+ cpes = Element('cpe-list')
+ cpes.set('xmlns:cpe-23', "http://scap.nist.gov/schema/cpe-extension/2.3")
+ cpes.set('xmlns:ns6', "http://scap.nist.gov/schema/scap-core/0.1")
+ cpes.set('xmlns:scap-core', "http://scap.nist.gov/schema/scap-core/0.3")
+ cpes.set('xmlns:config', "http://scap.nist.gov/schema/configuration/0.1")
+ cpes.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance")
+ cpes.set('xmlns:meta', "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2")
+ cpes.set('xsi:schemaLocation', " ".join(["http://scap.nist.gov/schema/cpe-extension/2.3",
+ "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd",
+ "http://cpe.mitre.org/dictionary/2.0",
+ "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd",
+ "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
+ "https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd",
+ "http://scap.nist.gov/schema/scap-core/0.3",
+ "https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd",
+ "http://scap.nist.gov/schema/configuration/0.1",
+ "https://scap.nist.gov/schema/nvd/configuration_0.1.xsd",
+ "http://scap.nist.gov/schema/scap-core/0.1",
+ "https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"]))
+ item = SubElement(cpes, 'cpe-item')
+ cpe_short_name = CPE.short_name(self.cpe_str)
+ cpe_new_ver = CPE.version_update(self.cpe_str)
+
+ item.set('name', 'cpe:/' + cpe_short_name)
+ self.titles[0].text.replace(self.cpe_cur_ver, cpe_new_ver)
+ for title in self.titles:
+ item.append(title)
+ if self.references:
+ item.append(self.references)
+ cpe23item = SubElement(item, 'cpe-23:cpe23-item')
+ cpe23item.set('name', self.cpe_str)
+
+ # Generate the XML as a string
+ xmlstr = ET.tostring(cpes)
+
+ # And use minidom to pretty print the XML
+ return minidom.parseString(xmlstr).toprettyxml(encoding="utf-8").decode("utf-8")
+
+ @staticmethod
+ def version(cpe):
+ return cpe.split(":")[5]
+
+ @staticmethod
+ def product(cpe):
+ return cpe.split(":")[4]
+
+ @staticmethod
+ def short_name(cpe):
+ return ":".join(cpe.split(":")[2:6])
+
+ @staticmethod
+ def version_update(cpe):
+ return ":".join(cpe.split(":")[5:6])
+
+ @staticmethod
+ def no_version(cpe):
+ return ":".join(cpe.split(":")[:5])
+
+
+class CPEDB:
+ def __init__(self, nvd_path):
+ self.all_cpes = dict()
+ self.all_cpes_no_version = dict()
+ self.nvd_path = nvd_path
+
+ def get_xml_dict(self):
+ print("CPE: Setting up NIST dictionary")
+ if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
+ os.makedirs(os.path.join(self.nvd_path, "cpe"))
+
+ cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
+ if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
+ print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
+ cpe_dict = requests.get(CPEDB_URL)
+ open(cpe_dict_local, "wb").write(cpe_dict.content)
+
+ print("CPE: Unzipping xml manifest...")
+ nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb'))
+ print("CPE: Converting xml manifest to dict...")
+ tree = ET.parse(nist_cpe_file)
+ all_cpedb = tree.getroot()
+ self.parse_dict(all_cpedb)
+
+ def parse_dict(self, all_cpedb):
+ # Cycle through the dict and build two dict to be used for custom
+ # lookups of partial and complete CPE objects
+ # The objects are then used to create new proposed XML updates if
+ # if is determined one is required
+ # Out of the different language titles, select English
+ for cpe in all_cpedb.findall(".//{http://cpe.mitre.org/dictionary/2.0}cpe-item"):
+ cpe_titles = []
+ for title in cpe.findall('.//{http://cpe.mitre.org/dictionary/2.0}title[@xml:lang="en-US"]', ns):
+ title.tail = None
+ cpe_titles.append(title)
+
+ # Some older CPE don't include references, if they do, make
+ # sure we handle the case of one ref needing to be packed
+ # in a list
+ cpe_ref = cpe.find(".//{http://cpe.mitre.org/dictionary/2.0}references")
+ if cpe_ref:
+ for ref in cpe_ref.findall(".//{http://cpe.mitre.org/dictionary/2.0}reference"):
+ ref.tail = None
+ ref.text = ref.text.upper()
+ if ref.text not in VALID_REFS:
+ ref.text = ref.text + "-- UPDATE this entry, here are some examples and just one word should be used -- " + ' '.join(VALID_REFS) # noqa E501
+ cpe_ref.tail = None
+ cpe_ref.text = None
+
+ cpe_str = cpe.find(".//{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item").get('name')
+ item = CPE(cpe_str, cpe_titles, cpe_ref)
+ cpe_str_no_version = CPE.no_version(cpe_str)
+ # This dict must have a unique key for every CPE version
+ # which allows matching to the specific obj data of that
+ # NIST dict entry
+ self.all_cpes.update({cpe_str: item})
+ # This dict has one entry for every CPE (w/o version) to allow
+ # partial match (no valid version) check (the obj is saved and
+ # used as seed for suggested xml updates. By updating the same
+ # non-version'd entry, it assumes the last update here is the
+ # latest version in the NIST dict)
+ self.all_cpes_no_version.update({cpe_str_no_version: item})
+
+ def find_partial(self, cpe_str):
+ cpe_str_no_version = CPE.no_version(cpe_str)
+ if cpe_str_no_version in self.all_cpes_no_version:
+ return cpe_str_no_version
+
+ def find_partial_obj(self, cpe_str):
+ cpe_str_no_version = CPE.no_version(cpe_str)
+ if cpe_str_no_version in self.all_cpes_no_version:
+ return self.all_cpes_no_version[cpe_str_no_version]
+
+ def find_partial_latest_version(self, cpe_str_partial):
+ cpe_obj = self.find_partial_obj(cpe_str_partial)
+ return cpe_obj.cpe_cur_ver
+
+ def find(self, cpe_str):
+ if self.find_partial(cpe_str):
+ if cpe_str in self.all_cpes:
+ return cpe_str
+
+ def gen_update_xml(self, cpe_str):
+ cpe = self.find_partial_obj(cpe_str)
+ return cpe.update_xml_dict()
diff --git a/support/scripts/cve-checker b/support/scripts/cve-checker
deleted file mode 100755
index 998ea5b8af..0000000000
--- a/support/scripts/cve-checker
+++ /dev/null
@@ -1,196 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2009 by Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
-# Copyright (C) 2020 by Gregory CLEMENT <gregory.clement@bootlin.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-import argparse
-import datetime
-import os
-import json
-import sys
-import cve as cvecheck
-
-
-class Package:
- def __init__(self, name, version, ignored_cves):
- self.name = name
- self.version = version
- self.cves = list()
- self.ignored_cves = ignored_cves
-
-
-def check_package_cves(nvd_path, packages):
- if not os.path.isdir(nvd_path):
- os.makedirs(nvd_path)
-
- for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
- for pkg_name in cve.pkg_names:
- pkg = packages.get(pkg_name, '')
- if pkg and cve.affects(pkg.name, pkg.version, pkg.ignored_cves) == cve.CVE_AFFECTS:
- pkg.cves.append(cve.identifier)
-
-
-html_header = """
-<head>
-<script src=\"https://www.kryogenix.org/code/browser/sorttable/sorttable.js\"></script>
-<style type=\"text/css\">
-table {
- width: 100%;
-}
-td {
- border: 1px solid black;
-}
-td.centered {
- text-align: center;
-}
-td.wrong {
- background: #ff9a69;
-}
-td.correct {
- background: #d2ffc4;
-}
-
-</style>
-<title>CVE status for Buildroot configuration</title>
-</head>
-
-<p id=\"sortable_hint\"></p>
-"""
-
-
-html_footer = """
-</body>
-<script>
-if (typeof sorttable === \"object\") {
- document.getElementById(\"sortable_hint\").innerHTML =
- \"hint: the table can be sorted by clicking the column headers\"
-}
-</script>
-</html>
-"""
-
-
-def dump_html_pkg(f, pkg):
- f.write(" <tr>\n")
- f.write(" <td>%s</td>\n" % pkg.name)
-
- # Current version
- if len(pkg.version) > 20:
- version = pkg.version[:20] + "..."
- else:
- version = pkg.version
- f.write(" <td class=\"centered\">%s</td>\n" % version)
-
- # CVEs
- td_class = ["centered"]
- if len(pkg.cves) == 0:
- td_class.append("correct")
- else:
- td_class.append("wrong")
- f.write(" <td class=\"%s\">\n" % " ".join(td_class))
- for cve in pkg.cves:
- f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
- f.write(" </td>\n")
-
- f.write(" </tr>\n")
-
-
-def dump_html_all_pkgs(f, packages):
- f.write("""
-<table class=\"sortable\">
-<tr>
-<td>Package</td>
-<td class=\"centered\">Version</td>
-<td class=\"centered\">CVEs</td>
-</tr>
-""")
- for pkg in packages:
- dump_html_pkg(f, pkg)
- f.write("</table>")
-
-
-def dump_html_gen_info(f, date):
- f.write("<p><i>Generated on %s</i></p>\n" % (str(date)))
-
-
-def dump_html(packages, date, output):
- with open(output, 'w') as f:
- f.write(html_header)
- dump_html_all_pkgs(f, packages)
- dump_html_gen_info(f, date)
- f.write(html_footer)
-
-
-def dump_json(packages, date, output):
- # Format packages as a dictionnary instead of a list
- pkgs = {
- pkg.name: {
- "version": pkg.version,
- "cves": pkg.cves,
- } for pkg in packages
- }
- # The actual structure to dump, add date to it
- final = {'packages': pkgs,
- 'date': str(date)}
- with open(output, 'w') as f:
- json.dump(final, f, indent=2, separators=(',', ': '))
- f.write('\n')
-
-
-def resolvepath(path):
- return os.path.abspath(os.path.expanduser(path))
-
-
-def parse_args():
- parser = argparse.ArgumentParser()
- output = parser.add_argument_group('output', 'Output file(s)')
- output.add_argument('--html', dest='html', type=resolvepath,
- help='HTML output file')
- output.add_argument('--json', dest='json', type=resolvepath,
- help='JSON output file')
- parser.add_argument('--nvd-path', dest='nvd_path',
- help='Path to the local NVD database', type=resolvepath,
- required=True)
- args = parser.parse_args()
- if not args.html and not args.json:
- parser.error('at least one of --html or --json (or both) is required')
- return args
-
-
-def __main__():
- packages = list()
- content = json.load(sys.stdin)
- for item in content:
- pkg = content[item]
- p = Package(item, pkg.get('version', ''), pkg.get('ignore_cves', ''))
- packages.append(p)
-
- args = parse_args()
- date = datetime.datetime.utcnow()
-
- print("Checking packages CVEs")
- check_package_cves(args.nvd_path, {p.name: p for p in packages})
-
- if args.html:
- print("Write HTML")
- dump_html(packages, date, args.html)
- if args.json:
- print("Write JSON")
- dump_json(packages, date, args.json)
-
-
-__main__()
diff --git a/support/scripts/cve.py b/support/scripts/cve.py
index 6396019e0e..6e97ea193f 100755
--- a/support/scripts/cve.py
+++ b/support/scripts/cve.py
@@ -47,6 +47,24 @@ ops = {
}
+# Check if two CPE IDs match each other
+def cpe_matches(cpe1, cpe2):
+ cpe1_elems = cpe1.split(":")
+ cpe2_elems = cpe2.split(":")
+
+ remains = filter(lambda x: x[0] not in ["*", "-"] and x[1] not in ["*", "-"] and x[0] != x[1],
+ zip(cpe1_elems, cpe2_elems))
+ return len(list(remains)) == 0
+
+
+def cpe_product(cpe):
+ return cpe.split(':')[4]
+
+
+def cpe_version(cpe):
+ return cpe.split(':')[5]
+
+
class CVE:
"""An accessor class for CVE Items in NVD files"""
CVE_AFFECTS = 1
@@ -134,7 +152,11 @@ class CVE:
for cpe in node.get('cpe_match', ()):
if not cpe['vulnerable']:
return
- vendor, product, version = cpe['cpe23Uri'].split(':')[3:6]
+ product = cpe_product(cpe['cpe23Uri'])
+ version = cpe_version(cpe['cpe23Uri'])
+ # ignore when product is '-', which means N/A
+ if product == '-':
+ return
op_start = ''
op_end = ''
v_start = ''
@@ -144,10 +166,6 @@ class CVE:
# Version is defined, this is a '=' match
op_start = '='
v_start = version
- elif version == '-':
- # no version information is available
- op_start = '='
- v_start = version
else:
# Parse start version, end version and operators
if 'versionStartIncluding' in cpe:
@@ -167,8 +185,7 @@ class CVE:
v_end = cpe['versionEndExcluding']
yield {
- 'vendor': vendor,
- 'product': product,
+ 'id': cpe['cpe23Uri'],
'v_start': v_start,
'op_start': op_start,
'v_end': v_end,
@@ -186,11 +203,11 @@ class CVE:
return self.nvd_cve['cve']['CVE_data_meta']['ID']
@property
- def pkg_names(self):
- """The set of package names referred by this CVE definition"""
- return set(p['product'] for p in self.each_cpe())
+ def affected_products(self):
+ """The set of CPE products referred by this CVE definition"""
+ return set(cpe_product(p['id']) for p in self.each_cpe())
- def affects(self, name, version, cve_ignore_list):
+ def affects(self, name, version, cve_ignore_list, cpeid=None):
"""
True if the Buildroot Package object passed as argument is affected
by this CVE.
@@ -203,14 +220,15 @@ class CVE:
print("Cannot parse package '%s' version '%s'" % (name, version))
pkg_version = None
+ # if we don't have a cpeid, build one based on name and version
+ if not cpeid:
+ cpeid = "cpe:2.3:*:*:%s:%s:*:*:*:*:*:*:*" % (name, version)
+
for cpe in self.each_cpe():
- if cpe['product'] != name:
+ if not cpe_matches(cpe['id'], cpeid):
continue
- if cpe['v_start'] == '-':
- return self.CVE_AFFECTS
if not cpe['v_start'] and not cpe['v_end']:
- print("No CVE affected version")
- continue
+ return self.CVE_AFFECTS
if not pkg_version:
continue
diff --git a/support/scripts/gen-bootlin-toolchains b/support/scripts/gen-bootlin-toolchains
index ac1a19b3c0..902b7795ee 100755
--- a/support/scripts/gen-bootlin-toolchains
+++ b/support/scripts/gen-bootlin-toolchains
@@ -123,6 +123,16 @@ arches = {
'conditions': ['BR2_or1k'],
'prefix': 'or1k',
},
+ 'powerpc-440fp': {
+ # Not sure it could be used by other powerpc variants?
+ 'conditions': ['BR2_powerpc', 'BR2_powerpc_440fp'],
+ 'prefix': 'powerpc',
+ },
+ 'powerpc-e300c3': {
+ # Not sure it could be used by other powerpc variants?
+ 'conditions': ['BR2_powerpc', 'BR2_powerpc_e300c3'],
+ 'prefix': 'powerpc',
+ },
'powerpc-e500mc': {
# Not sure it could be used by other powerpc variants?
'conditions': ['BR2_powerpc', 'BR2_powerpc_e500mc'],
@@ -132,6 +142,10 @@ arches = {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_e5500'],
'prefix': 'powerpc64',
},
+ 'powerpc64-e6500': {
+ 'conditions': ['BR2_powerpc64', 'BR2_powerpc_e6500'],
+ 'prefix': 'powerpc64',
+ },
'powerpc64-power8': {
'conditions': ['BR2_powerpc64', 'BR2_powerpc_power8'],
'prefix': 'powerpc64',
@@ -196,7 +210,7 @@ arches = {
'prefix': 'i686',
},
'xtensa-lx60': {
- 'conditions': ['BR2_xtensa', 'BR2_xtensa_fsf'],
+ 'conditions': ['BR2_xtensa', 'BR2_XTENSA_CUSTOM', 'BR2_XTENSA_LITTLE_ENDIAN'],
'prefix': 'xtensa',
},
}
@@ -233,18 +247,25 @@ class Toolchain:
f.write("config %s\n" % self.option_name)
f.write("\tbool \"%s %s %s %s\"\n" %
(self.arch, self.libc, self.variant, self.version))
- for c in arches[self.arch]['conditions']:
- f.write("\tdepends on %s\n" % c)
+ depends = []
selects = []
+
+ for c in arches[self.arch]['conditions']:
+ depends.append(c)
+
for frag in self.fragment:
# libc type
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_UCLIBC"):
selects.append("BR2_TOOLCHAIN_EXTERNAL_UCLIBC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_GLIBC"):
+ # glibc needs mmu support
+ depends.append("BR2_USE_MMU")
+ # glibc doesn't support static only configuration
+ depends.append("!BR2_STATIC_LIBS")
selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
- # all glibc toolchains have RPC support
- selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
+ # musl needs mmu support
+ depends.append("BR2_USE_MMU")
selects.append("BR2_TOOLCHAIN_EXTERNAL_MUSL")
# gcc version
@@ -303,6 +324,9 @@ class Toolchain:
if frag.startswith("BR2_TOOLCHAIN_EXTERNAL_OPENMP"):
selects.append("BR2_TOOLCHAIN_HAS_OPENMP")
+ for depend in depends:
+ f.write("\tdepends on %s\n" % depend)
+
for select in selects:
f.write("\tselect %s\n" % select)
diff --git a/support/scripts/generate-gitlab-ci-yml b/support/scripts/generate-gitlab-ci-yml
index e42943953c..3f498e08fd 100755
--- a/support/scripts/generate-gitlab-ci-yml
+++ b/support/scripts/generate-gitlab-ci-yml
@@ -6,9 +6,6 @@ main() {
local template="${1}"
preamble "${template}"
-
- gen_basics
- gen_defconfigs
gen_tests
}
@@ -24,91 +21,85 @@ preamble() {
_EOF_
}
-gen_basics() {
- local tst
-
- # Skip basic tests when explicitly building defconfigs or runtime tests
- case "${CI_COMMIT_REF_NAME}" in
- (*-defconfigs) return;;
- (*-*_defconfig) return;;
- (*-runtime-tests) return;;
- (*-tests.*) return;;
- esac
-
- for tst in DEVELOPERS flake8 package; do
- printf 'check-%s: { extends: .check-%s_base }\n' "${tst}" "${tst}"
- done
-}
+gen_tests() {
+ local -a basics defconfigs runtimes
+ local do_basics do_defconfigs do_runtime
+ local defconfigs_ext cfg tst
-gen_defconfigs() {
- local -a defconfigs
- local template cfg ext
+ basics=( DEVELOPERS flake8 package )
defconfigs=( $(cd configs; LC_ALL=C ls -1 *_defconfig) )
+ runtimes=( $(./support/testing/run-tests -l 2>&1 \
+ | sed -r -e '/^test_run \((.*)\).*/!d; s//\1/' \
+ | LC_ALL=C sort)
+ )
+
if [ -n "${CI_COMMIT_TAG}" ]; then
- # For tags, create a pipeline.
- template=base
+ do_basics=true
+ do_defconfigs=base
+ do_runtime=true
+ elif [ "${CI_PIPELINE_SOURCE}" = "trigger" ]; then
+ case "${BR_SCHEDULE_JOBS}" in
+ (basic)
+ do_basics=true
+ do_defconfigs=check
+ defconfigs_ext=_check
+ ;;
+ (defconfig)
+ do_defconfigs=base
+ ;;
+ (runtime)
+ do_runtime=true
+ ;;
+ esac
+ else
+ case "${CI_COMMIT_REF_NAME}" in
+ (*-basics)
+ do_basics=true
+ do_defconfigs=check
+ defconfigs_ext=_check
+ ;;
+ (*-defconfigs)
+ do_defconfigs=base
+ ;;
+ (*-*_defconfig)
+ defconfigs=( "${CI_COMMIT_REF_NAME##*-}" )
+ do_defconfigs=base
+ ;;
+ (*-runtime-tests)
+ do_runtime=true
+ ;;
+ (*-tests.*)
+ runtimes=( "${CI_COMMIT_REF_NAME##*-}" )
+ do_runtime=true
+ ;;
+ esac
fi
- if [ -n "${CI_PIPELINE_TRIGGERED}" ]; then
- # For pipeline created by using a trigger token.
- template=base
+
+ # If nothing else, at least do the basics to generate a valid pipeline
+ if [ -z "${do_defconfigs}" \
+ -a -z "${do_runtime}" \
+ ]
+ then
+ do_basics=true
fi
- case "${CI_COMMIT_REF_NAME}" in
- # For master, next, and maintenance branches, only check the defconfigs
- (master|next|????.??.x)
- template=check
- ext=_check
- ;;
- # For the branch or tag name named *-defconfigs, create a pipeline.
- (*-defconfigs)
- template=base
- ;;
- (*-*_defconfig)
- defconfigs=( "${CI_COMMIT_REF_NAME##*-}" )
- template=base
- ;;
- esac
- if [ -n "${template}" ]; then
+ if ${do_basics:-false}; then
+ for tst in "${basics[@]}"; do
+ printf 'check-%s: { extends: .check-%s_base }\n' "${tst}" "${tst}"
+ done
+ fi
+
+ if [ -n "${do_defconfigs}" ]; then
for cfg in "${defconfigs[@]}"; do
printf '%s%s: { extends: .defconfig_%s }\n' \
- "${cfg}" "${ext}" "${template}"
+ "${cfg}" "${defconfigs_ext}" "${do_defconfigs}"
done
fi
-}
-
-gen_tests() {
- local -a tests
- local run_tests tst
- tests=( $(./support/testing/run-tests -l 2>&1 \
- | sed -r -e '/^test_run \((.*)\).*/!d; s//\1/'\
- | LC_ALL=C sort)
- )
-
- run_tests=false
- if [ -n "${CI_COMMIT_TAG}" ]; then
- # For tags, create a pipeline.
- run_tests=true
- fi
- if [ -n "${CI_PIPELINE_TRIGGERED}" ]; then
- # For pipeline created by using a trigger token.
- run_tests=true
- fi
- case "${CI_COMMIT_REF_NAME}" in
- # For the branch or tag name named *-runtime-tests, create a pipeline.
- (*-runtime-tests)
- run_tests=true
- ;;
- (*-tests.*)
- tests=( "${CI_COMMIT_REF_NAME##*-}" )
- run_tests=true
- ;;
- esac
-
- if ${run_tests}; then
- printf '%s: { extends: .runtime_test_base }\n' "${tests[@]}"
+ if ${do_runtime:-false}; then
+ printf '%s: { extends: .runtime_test_base }\n' "${runtimes[@]}"
fi
}
diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats
index 503cc45c16..6f3ddc561f 100755
--- a/support/scripts/pkg-stats
+++ b/support/scripts/pkg-stats
@@ -28,10 +28,11 @@ import subprocess
import json
import sys
-sys.path.append('utils/')
-from getdeveloperlib import parse_developers # noqa: E402
-import cve as cvecheck # noqa: E402
+brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
+sys.path.append(os.path.join(brpath, "utils"))
+from getdeveloperlib import parse_developers # noqa: E402
+from cpedb import CPEDB # noqa: E402
INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
URL_RE = re.compile(r"\s*https?://\S*\s*$")
@@ -66,7 +67,7 @@ def get_defconfig_list():
"""
return [
Defconfig(name[:-len('_defconfig')], os.path.join('configs', name))
- for name in os.listdir('configs')
+ for name in os.listdir(os.path.join(brpath, 'configs'))
if name.endswith('_defconfig')
]
@@ -76,6 +77,7 @@ class Package:
all_license_files = list()
all_versions = dict()
all_ignored_cves = dict()
+ all_cpeids = dict()
# This is the list of all possible checks. Add new checks to this list so
# a tool that post-processeds the json output knows the checks before
# iterating over the packages.
@@ -96,7 +98,9 @@ class Package:
self.current_version = None
self.url = None
self.url_worker = None
+ self.cpeid = None
self.cves = list()
+ self.ignored_cves = list()
self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None}
self.status = {}
@@ -108,9 +112,10 @@ class Package:
Fills in the .url field
"""
self.status['url'] = ("warning", "no Config.in")
- for filename in os.listdir(os.path.dirname(self.path)):
+ pkgdir = os.path.dirname(os.path.join(brpath, self.path))
+ for filename in os.listdir(pkgdir):
if fnmatch.fnmatch(filename, 'Config.*'):
- fp = open(os.path.join(os.path.dirname(self.path), filename), "r")
+ fp = open(os.path.join(pkgdir, filename), "r")
for config_line in fp:
if URL_RE.match(config_line):
self.url = config_line.strip()
@@ -138,10 +143,10 @@ class Package:
Fills in the .infras field
"""
self.infras = list()
- with open(self.path, 'r') as f:
+ with open(os.path.join(brpath, self.path), 'r') as f:
lines = f.readlines()
- for l in lines:
- match = INFRA_RE.match(l)
+ for line in lines:
+ match = INFRA_RE.match(line)
if not match:
continue
infra = match.group(1)
@@ -178,7 +183,7 @@ class Package:
return
hashpath = self.path.replace(".mk", ".hash")
- if os.path.exists(hashpath):
+ if os.path.exists(os.path.join(brpath, hashpath)):
self.status['hash'] = ("ok", "found")
else:
self.status['hash'] = ("error", "missing")
@@ -191,7 +196,7 @@ class Package:
self.status['patches'] = ("na", "no valid package infra")
return
- pkgdir = os.path.dirname(self.path)
+ pkgdir = os.path.dirname(os.path.join(brpath, self.path))
for subdir, _, _ in os.walk(pkgdir):
self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch')
@@ -210,12 +215,28 @@ class Package:
if var in self.all_versions:
self.current_version = self.all_versions[var]
+ def set_cpeid(self):
+ """
+ Fills in the .cpeid field
+ """
+ var = self.pkgvar()
+ if not self.has_valid_infra:
+ self.status['cpe'] = ("na", "no valid package infra")
+ return
+
+ if var in self.all_cpeids:
+ self.cpeid = self.all_cpeids[var]
+ # Set a preliminary status, it might be overridden by check_package_cpes()
+ self.status['cpe'] = ("warning", "not checked against CPE dictionnary")
+ else:
+ self.status['cpe'] = ("error", "no verified CPE identifier")
+
def set_check_package_warnings(self):
"""
Fills in the .warnings and .status['pkg-check'] fields
"""
- cmd = ["./utils/check-package"]
- pkgdir = os.path.dirname(self.path)
+ cmd = [os.path.join(brpath, "utils/check-package")]
+ pkgdir = os.path.dirname(os.path.join(brpath, self.path))
self.status['pkg-check'] = ("error", "Missing")
for root, dirs, files in os.walk(pkgdir):
for f in files:
@@ -233,12 +254,11 @@ class Package:
self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings))
return
- @property
- def ignored_cves(self):
+ def set_ignored_cves(self):
"""
Give the list of CVEs ignored by the package
"""
- return list(self.all_ignored_cves.get(self.pkgvar(), []))
+ self.ignored_cves = list(self.all_ignored_cves.get(self.pkgvar(), []))
def set_developers(self, developers):
"""
@@ -256,7 +276,13 @@ class Package:
self.status['developers'] = ("warning", "no developers")
def is_status_ok(self, name):
- return self.status[name][0] == 'ok'
+ return name in self.status and self.status[name][0] == 'ok'
+
+ def is_status_error(self, name):
+ return name in self.status and self.status[name][0] == 'error'
+
+ def is_status_na(self, name):
+ return name in self.status and self.status[name][0] == 'na'
def __eq__(self, other):
return self.path == other.path
@@ -300,11 +326,12 @@ def get_pkglist(npackages, package_list):
"toolchain/toolchain-wrapper.mk"]
packages = list()
count = 0
- for root, dirs, files in os.walk("."):
+ for root, dirs, files in os.walk(brpath):
+ root = os.path.relpath(root, brpath)
rootdir = root.split("/")
- if len(rootdir) < 2:
+ if len(rootdir) < 1:
continue
- if rootdir[1] not in WALK_USEFUL_SUBDIRS:
+ if rootdir[0] not in WALK_USEFUL_SUBDIRS:
continue
for f in files:
if not f.endswith(".mk"):
@@ -316,8 +343,7 @@ def get_pkglist(npackages, package_list):
pkgpath = os.path.join(root, f)
skip = False
for exclude in WALK_EXCLUDES:
- # pkgpath[2:] strips the initial './'
- if re.match(exclude, pkgpath[2:]):
+ if re.match(exclude, pkgpath):
skip = True
continue
if skip:
@@ -330,10 +356,16 @@ def get_pkglist(npackages, package_list):
return packages
+def get_config_packages():
+ cmd = ["make", "--no-print-directory", "show-info"]
+ js = json.loads(subprocess.check_output(cmd))
+ return set([v["name"] for v in js.values()])
+
+
def package_init_make_info():
# Fetch all variables at once
variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars",
- "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES"])
+ "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES %_CPE_ID"])
variable_list = variables.decode().splitlines()
# We process first the host package VERSION, and then the target
@@ -371,6 +403,10 @@ def package_init_make_info():
pkgvar = pkgvar[:-12]
Package.all_ignored_cves[pkgvar] = value.split()
+ elif pkgvar.endswith("_CPE_ID"):
+ pkgvar = pkgvar[:-7]
+ Package.all_cpeids[pkgvar] = value
+
check_url_count = 0
@@ -405,7 +441,7 @@ async def check_package_urls(packages):
async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
packages = [p for p in packages if p.status['url'][0] == 'ok']
for pkg in packages:
- tasks.append(check_url_status(sess, pkg, len(packages)))
+ tasks.append(asyncio.ensure_future(check_url_status(sess, pkg, len(packages))))
await asyncio.wait(tasks)
@@ -523,20 +559,58 @@ async def check_package_latest_version(packages):
async with aiohttp.ClientSession(connector=connector, trust_env=True) as sess:
packages = [p for p in packages if p.has_valid_infra]
for pkg in packages:
- tasks.append(check_package_latest_version_get(sess, pkg, len(packages)))
+ tasks.append(asyncio.ensure_future(check_package_latest_version_get(sess, pkg, len(packages))))
await asyncio.wait(tasks)
+def check_package_cve_affects(cve, cpe_product_pkgs):
+ for product in cve.affected_products:
+ if product not in cpe_product_pkgs:
+ continue
+ for pkg in cpe_product_pkgs[product]:
+ if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves, pkg.cpeid) == cve.CVE_AFFECTS:
+ pkg.cves.append(cve.identifier)
+
+
def check_package_cves(nvd_path, packages):
if not os.path.isdir(nvd_path):
os.makedirs(nvd_path)
+ cpe_product_pkgs = defaultdict(list)
+ for pkg in packages:
+ if not pkg.has_valid_infra:
+ pkg.status['cve'] = ("na", "no valid package infra")
+ continue
+ if not pkg.current_version:
+ pkg.status['cve'] = ("na", "no version information available")
+ continue
+ if pkg.cpeid:
+ cpe_product = cvecheck.cpe_product(pkg.cpeid)
+ cpe_product_pkgs[cpe_product].append(pkg)
+ else:
+ cpe_product_pkgs[pkg.name].append(pkg)
+
for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
- for pkg_name in cve.pkg_names:
- if pkg_name in packages:
- pkg = packages[pkg_name]
- if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves) == cve.CVE_AFFECTS:
- pkg.cves.append(cve.identifier)
+ check_package_cve_affects(cve, cpe_product_pkgs)
+
+ for pkg in packages:
+ if 'cve' not in pkg.status:
+ if pkg.cves:
+ pkg.status['cve'] = ("error", "affected by CVEs")
+ else:
+ pkg.status['cve'] = ("ok", "not affected by CVEs")
+
+
+def check_package_cpes(nvd_path, packages):
+ cpedb = CPEDB(nvd_path)
+ cpedb.get_xml_dict()
+ for p in packages:
+ if not p.cpeid:
+ continue
+ if cpedb.find(p.cpeid):
+ p.status['cpe'] = ("ok", "verified CPE identifier")
+ else:
+ p.status['cpe'] = ("error", "CPE identifier unknown in CPE database")
def calculate_stats(packages):
@@ -578,6 +652,10 @@ def calculate_stats(packages):
stats["total-cves"] += len(pkg.cves)
if len(pkg.cves) != 0:
stats["pkg-cves"] += 1
+ if pkg.cpeid:
+ stats["cpe-id"] += 1
+ else:
+ stats["no-cpe-id"] += 1
return stats
@@ -633,6 +711,30 @@ td.version-error {
background: #ccc;
}
+td.cpe-ok {
+ background: #d2ffc4;
+}
+
+td.cpe-nok {
+ background: #ff9a69;
+}
+
+td.cpe-unknown {
+ background: #ffd870;
+}
+
+td.cve-ok {
+ background: #d2ffc4;
+}
+
+td.cve-nok {
+ background: #ff9a69;
+}
+
+td.cve-unknown {
+ background: #ffd870;
+}
+
</style>
<title>Statistics of Buildroot packages</title>
</head>
@@ -678,7 +780,7 @@ def boolean_str(b):
def dump_html_pkg(f, pkg):
f.write(" <tr>\n")
- f.write(" <td>%s</td>\n" % pkg.path[2:])
+ f.write(" <td>%s</td>\n" % pkg.path)
# Patch count
td_class = ["centered"]
@@ -791,13 +893,35 @@ def dump_html_pkg(f, pkg):
# CVEs
td_class = ["centered"]
- if len(pkg.cves) == 0:
- td_class.append("correct")
+ if pkg.is_status_ok("cve"):
+ td_class.append("cve-ok")
+ elif pkg.is_status_error("cve"):
+ td_class.append("cve-nok")
else:
- td_class.append("wrong")
+ td_class.append("cve-unknown")
+ f.write(" <td class=\"%s\">\n" % " ".join(td_class))
+ if pkg.is_status_error("cve"):
+ for cve in pkg.cves:
+ f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
+ elif pkg.is_status_na("cve"):
+ f.write(" %s" % pkg.status['cve'][1])
+ else:
+ f.write(" N/A\n")
+ f.write(" </td>\n")
+
+ # CPE ID
+ td_class = ["left"]
+ if pkg.is_status_ok("cpe"):
+ td_class.append("cpe-ok")
+ elif pkg.is_status_error("cpe"):
+ td_class.append("cpe-nok")
+ else:
+ td_class.append("cpe-unknown")
f.write(" <td class=\"%s\">\n" % " ".join(td_class))
- for cve in pkg.cves:
- f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
+ if pkg.cpeid:
+ f.write(" <code>%s</code>\n" % pkg.cpeid)
+ if not pkg.is_status_ok("cpe"):
+ f.write(" %s%s\n" % ("<br/>" if pkg.cpeid else "", pkg.status['cpe'][1]))
f.write(" </td>\n")
f.write(" </tr>\n")
@@ -818,6 +942,7 @@ def dump_html_all_pkgs(f, packages):
<td class=\"centered\">Warnings</td>
<td class=\"centered\">Upstream URL</td>
<td class=\"centered\">CVEs</td>
+<td class=\"centered\">CPE ID</td>
</tr>
""")
for pkg in sorted(packages):
@@ -860,6 +985,10 @@ def dump_html_stats(f, stats):
stats["pkg-cves"])
f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" %
stats["total-cves"])
+ f.write("<tr><td>Packages with CPE ID</td><td>%s</td></tr>\n" %
+ stats["cpe-id"])
+ f.write("<tr><td>Packages without CPE ID</td><td>%s</td></tr>\n" %
+ stats["no-cpe-id"])
f.write("</table>\n")
@@ -926,6 +1055,8 @@ def parse_args():
output.add_argument('--json', dest='json', type=resolvepath,
help='JSON output file')
packages = parser.add_mutually_exclusive_group()
+ packages.add_argument('-c', dest='configpackages', action='store_true',
+ help='Apply to packages enabled in current configuration')
packages.add_argument('-n', dest='npackages', type=int, action='store',
help='Number of packages')
packages.add_argument('-p', dest='packages', action='store',
@@ -939,13 +1070,22 @@ def parse_args():
def __main__():
+ global cvecheck
+
args = parse_args()
+
+ if args.nvd_path:
+ import cve as cvecheck
+
if args.packages:
package_list = args.packages.split(",")
+ elif args.configpackages:
+ package_list = get_config_packages()
else:
package_list = None
date = datetime.datetime.utcnow()
- commit = subprocess.check_output(['git', 'rev-parse',
+ commit = subprocess.check_output(['git', '-C', brpath,
+ 'rev-parse',
'HEAD']).splitlines()[0].decode()
print("Build package list ...")
packages = get_pkglist(args.npackages, package_list)
@@ -965,7 +1105,9 @@ def __main__():
pkg.set_patch_count()
pkg.set_check_package_warnings()
pkg.set_current_version()
+ pkg.set_cpeid()
pkg.set_url()
+ pkg.set_ignored_cves()
pkg.set_developers(developers)
print("Checking URL status")
loop = asyncio.get_event_loop()
@@ -975,7 +1117,8 @@ def __main__():
loop.run_until_complete(check_package_latest_version(packages))
if args.nvd_path:
print("Checking packages CVEs")
- check_package_cves(args.nvd_path, {p.name: p for p in packages})
+ check_package_cves(args.nvd_path, packages)
+ check_package_cpes(args.nvd_path, packages)
print("Calculate stats")
stats = calculate_stats(packages)
if args.html:
diff --git a/support/scripts/setlocalversion b/support/scripts/setlocalversion
index e04c955d9e..d492f2db2f 100755
--- a/support/scripts/setlocalversion
+++ b/support/scripts/setlocalversion
@@ -48,14 +48,28 @@ if head=`git rev-parse --verify --short HEAD 2>/dev/null`; then
fi
# Check for mercurial and a mercurial repo.
+# In the git case, 'git describe' will show the latest tag, and unless we are
+# exactly on that tag, the number of commits since then, and last commit id.
+# Mimic something similar in the Mercurial case.
if hgid=`HGRCPATH= hg id --id --tags 2>/dev/null`; then
tag=`printf '%s' "$hgid" | cut -d' ' -f2 --only-delimited`
# Do we have an untagged version?
if [ -z "$tag" -o "$tag" = tip ]; then
+ # current revision is not tagged, determine latest tag
+ latesttag=`HGRCPATH= hg log -r. -T '{latesttag}' 2>/dev/null`
+ # In case there is more than one tag on the latest tagged commit,
+ # 'latesttag' will separate them by colon (:). We'll retain this.
+ # In case there is no tag at all, 'null' will be returned.
+ if [ "$latesttag" = "null" ]; then
+ latesttag=''
+ fi
+
+ # add the commit id
id=`printf '%s' "$hgid" | sed 's/[+ ].*//'`
- printf '%s%s' -hg "$id"
+ printf '%s%s%s' "${latesttag}" -hg "$id"
else
+ # current revision is tagged, just print the tag
printf ${tag}
fi