aboutsummaryrefslogtreecommitdiff
path: root/support/scripts/pkg-stats
diff options
context:
space:
mode:
Diffstat (limited to 'support/scripts/pkg-stats')
-rwxr-xr-xsupport/scripts/pkg-stats183
1 files changed, 154 insertions, 29 deletions
diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats
index 503cc45c16..4a9ff1ffa0 100755
--- a/support/scripts/pkg-stats
+++ b/support/scripts/pkg-stats
@@ -28,7 +28,9 @@ import subprocess
import json
import sys
-sys.path.append('utils/')
+brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
+
+sys.path.append(os.path.join(brpath, "utils"))
from getdeveloperlib import parse_developers # noqa: E402
import cve as cvecheck # noqa: E402
@@ -66,7 +68,7 @@ def get_defconfig_list():
"""
return [
Defconfig(name[:-len('_defconfig')], os.path.join('configs', name))
- for name in os.listdir('configs')
+ for name in os.listdir(os.path.join(brpath, 'configs'))
if name.endswith('_defconfig')
]
@@ -76,6 +78,7 @@ class Package:
all_license_files = list()
all_versions = dict()
all_ignored_cves = dict()
+ all_cpeids = dict()
# This is the list of all possible checks. Add new checks to this list so
# a tool that post-processeds the json output knows the checks before
# iterating over the packages.
@@ -96,6 +99,7 @@ class Package:
self.current_version = None
self.url = None
self.url_worker = None
+ self.cpeid = None
self.cves = list()
self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None}
self.status = {}
@@ -108,9 +112,10 @@ class Package:
Fills in the .url field
"""
self.status['url'] = ("warning", "no Config.in")
- for filename in os.listdir(os.path.dirname(self.path)):
+ pkgdir = os.path.dirname(os.path.join(brpath, self.path))
+ for filename in os.listdir(pkgdir):
if fnmatch.fnmatch(filename, 'Config.*'):
- fp = open(os.path.join(os.path.dirname(self.path), filename), "r")
+ fp = open(os.path.join(pkgdir, filename), "r")
for config_line in fp:
if URL_RE.match(config_line):
self.url = config_line.strip()
@@ -138,7 +143,7 @@ class Package:
Fills in the .infras field
"""
self.infras = list()
- with open(self.path, 'r') as f:
+ with open(os.path.join(brpath, self.path), 'r') as f:
lines = f.readlines()
for l in lines:
match = INFRA_RE.match(l)
@@ -178,7 +183,7 @@ class Package:
return
hashpath = self.path.replace(".mk", ".hash")
- if os.path.exists(hashpath):
+ if os.path.exists(os.path.join(brpath, hashpath)):
self.status['hash'] = ("ok", "found")
else:
self.status['hash'] = ("error", "missing")
@@ -191,7 +196,7 @@ class Package:
self.status['patches'] = ("na", "no valid package infra")
return
- pkgdir = os.path.dirname(self.path)
+ pkgdir = os.path.dirname(os.path.join(brpath, self.path))
for subdir, _, _ in os.walk(pkgdir):
self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch')
@@ -210,12 +215,27 @@ class Package:
if var in self.all_versions:
self.current_version = self.all_versions[var]
+ def set_cpeid(self):
+ """
+ Fills in the .cpeid field
+ """
+ var = self.pkgvar()
+ if not self.has_valid_infra:
+ self.status['cpe'] = ("na", "no valid package infra")
+ return
+
+ if var in self.all_cpeids:
+ self.cpeid = self.all_cpeids[var]
+ self.status['cpe'] = ("ok", "verified CPE identifier")
+ else:
+ self.status['cpe'] = ("error", "no verified CPE identifier")
+
def set_check_package_warnings(self):
"""
Fills in the .warnings and .status['pkg-check'] fields
"""
- cmd = ["./utils/check-package"]
- pkgdir = os.path.dirname(self.path)
+ cmd = [os.path.join(brpath, "utils/check-package")]
+ pkgdir = os.path.dirname(os.path.join(brpath, self.path))
self.status['pkg-check'] = ("error", "Missing")
for root, dirs, files in os.walk(pkgdir):
for f in files:
@@ -300,11 +320,12 @@ def get_pkglist(npackages, package_list):
"toolchain/toolchain-wrapper.mk"]
packages = list()
count = 0
- for root, dirs, files in os.walk("."):
+ for root, dirs, files in os.walk(brpath):
+ root = os.path.relpath(root, brpath)
rootdir = root.split("/")
- if len(rootdir) < 2:
+ if len(rootdir) < 1:
continue
- if rootdir[1] not in WALK_USEFUL_SUBDIRS:
+ if rootdir[0] not in WALK_USEFUL_SUBDIRS:
continue
for f in files:
if not f.endswith(".mk"):
@@ -316,8 +337,7 @@ def get_pkglist(npackages, package_list):
pkgpath = os.path.join(root, f)
skip = False
for exclude in WALK_EXCLUDES:
- # pkgpath[2:] strips the initial './'
- if re.match(exclude, pkgpath[2:]):
+ if re.match(exclude, pkgpath):
skip = True
continue
if skip:
@@ -330,10 +350,16 @@ def get_pkglist(npackages, package_list):
return packages
+def get_config_packages():
+ cmd = ["make", "--no-print-directory", "show-info"]
+ js = json.loads(subprocess.check_output(cmd))
+ return js.keys()
+
+
def package_init_make_info():
# Fetch all variables at once
variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars",
- "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES"])
+ "VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES %_CPE_ID"])
variable_list = variables.decode().splitlines()
# We process first the host package VERSION, and then the target
@@ -371,6 +397,10 @@ def package_init_make_info():
pkgvar = pkgvar[:-12]
Package.all_ignored_cves[pkgvar] = value.split()
+ elif pkgvar.endswith("_CPE_ID"):
+ pkgvar = pkgvar[:-7]
+ Package.all_cpeids[pkgvar] = value
+
check_url_count = 0
@@ -527,16 +557,42 @@ async def check_package_latest_version(packages):
await asyncio.wait(tasks)
+def check_package_cve_affects(cve, cpe_product_pkgs):
+ for product in cve.affected_products:
+ if product not in cpe_product_pkgs:
+ continue
+ for pkg in cpe_product_pkgs[product]:
+ if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves, pkg.cpeid) == cve.CVE_AFFECTS:
+ pkg.cves.append(cve.identifier)
+
+
def check_package_cves(nvd_path, packages):
if not os.path.isdir(nvd_path):
os.makedirs(nvd_path)
+ cpe_product_pkgs = defaultdict(list)
+ for pkg in packages:
+ if not pkg.has_valid_infra:
+ pkg.status['cve'] = ("na", "no valid package infra")
+ continue
+ if not pkg.current_version:
+ pkg.status['cve'] = ("na", "no version information available")
+ continue
+ if pkg.cpeid:
+ cpe_product = cvecheck.cpe_product(pkg.cpeid)
+ cpe_product_pkgs[cpe_product].append(pkg)
+ else:
+ cpe_product_pkgs[pkg.name].append(pkg)
+
for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
- for pkg_name in cve.pkg_names:
- if pkg_name in packages:
- pkg = packages[pkg_name]
- if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves) == cve.CVE_AFFECTS:
- pkg.cves.append(cve.identifier)
+ check_package_cve_affects(cve, cpe_product_pkgs)
+
+ for pkg in packages:
+ if 'cve' not in pkg.status:
+ if pkg.cves:
+ pkg.status['cve'] = ("error", "affected by CVEs")
+ else:
+ pkg.status['cve'] = ("ok", "not affected by CVEs")
def calculate_stats(packages):
@@ -578,6 +634,10 @@ def calculate_stats(packages):
stats["total-cves"] += len(pkg.cves)
if len(pkg.cves) != 0:
stats["pkg-cves"] += 1
+ if pkg.cpeid:
+ stats["cpe-id"] += 1
+ else:
+ stats["no-cpe-id"] += 1
return stats
@@ -633,6 +693,30 @@ td.version-error {
background: #ccc;
}
+td.cpe-ok {
+ background: #d2ffc4;
+}
+
+td.cpe-nok {
+ background: #ff9a69;
+}
+
+td.cpe-unknown {
+ background: #ffd870;
+}
+
+td.cve-ok {
+ background: #d2ffc4;
+}
+
+td.cve-nok {
+ background: #ff9a69;
+}
+
+td.cve-unknown {
+ background: #ffd870;
+}
+
</style>
<title>Statistics of Buildroot packages</title>
</head>
@@ -678,7 +762,7 @@ def boolean_str(b):
def dump_html_pkg(f, pkg):
f.write(" <tr>\n")
- f.write(" <td>%s</td>\n" % pkg.path[2:])
+ f.write(" <td>%s</td>\n" % pkg.path)
# Patch count
td_class = ["centered"]
@@ -791,13 +875,35 @@ def dump_html_pkg(f, pkg):
# CVEs
td_class = ["centered"]
- if len(pkg.cves) == 0:
- td_class.append("correct")
+ if pkg.status['cve'][0] == "ok":
+ td_class.append("cve-ok")
+ elif pkg.status['cve'][0] == "error":
+ td_class.append("cve-nok")
else:
- td_class.append("wrong")
+ td_class.append("cve-unknown")
+ f.write(" <td class=\"%s\">\n" % " ".join(td_class))
+ if pkg.status['cve'][0] == "error":
+ for cve in pkg.cves:
+ f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
+ elif pkg.status['cve'][0] == "na":
+ f.write(" %s" % pkg.status['cve'][1])
+ f.write(" </td>\n")
+
+ # CPE ID
+ td_class = ["left"]
+ if pkg.status['cpe'][0] == "ok":
+ td_class.append("cpe-ok")
+ elif pkg.status['cpe'][0] == "error":
+ td_class.append("cpe-nok")
+ else:
+ td_class.append("cpe-unknown")
f.write(" <td class=\"%s\">\n" % " ".join(td_class))
- for cve in pkg.cves:
- f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
+ if pkg.status['cpe'][0] == "ok":
+ f.write(" <code>%s</code>\n" % pkg.cpeid)
+ elif pkg.status['cpe'][0] == "error":
+ f.write(" N/A\n")
+ else:
+ f.write(" %s\n" % pkg.status['cpe'][1])
f.write(" </td>\n")
f.write(" </tr>\n")
@@ -818,6 +924,7 @@ def dump_html_all_pkgs(f, packages):
<td class=\"centered\">Warnings</td>
<td class=\"centered\">Upstream URL</td>
<td class=\"centered\">CVEs</td>
+<td class=\"centered\">CPE ID</td>
</tr>
""")
for pkg in sorted(packages):
@@ -860,6 +967,10 @@ def dump_html_stats(f, stats):
stats["pkg-cves"])
f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" %
stats["total-cves"])
+ f.write("<tr><td>Packages with CPE ID</td><td>%s</td></tr>\n" %
+ stats["cpe-id"])
+ f.write("<tr><td>Packages without CPE ID</td><td>%s</td></tr>\n" %
+ stats["no-cpe-id"])
f.write("</table>\n")
@@ -926,31 +1037,44 @@ def parse_args():
output.add_argument('--json', dest='json', type=resolvepath,
help='JSON output file')
packages = parser.add_mutually_exclusive_group()
+ packages.add_argument('-c', dest='configpackages', action='store_true',
+ help='Apply to packages enabled in current configuration')
packages.add_argument('-n', dest='npackages', type=int, action='store',
help='Number of packages')
packages.add_argument('-p', dest='packages', action='store',
help='List of packages (comma separated)')
parser.add_argument('--nvd-path', dest='nvd_path',
help='Path to the local NVD database', type=resolvepath)
+ parser.add_argument("--cpeid", action='store_true')
args = parser.parse_args()
if not args.html and not args.json:
parser.error('at least one of --html or --json (or both) is required')
return args
+def cpeid_name(pkg):
+ try:
+ return pkg.cpeid.split(':')[1]
+ except Exception: # cpeid may be None, or improperly formatted
+ return ''
+
+
def __main__():
args = parse_args()
if args.packages:
package_list = args.packages.split(",")
+ elif args.configpackages:
+ package_list = get_config_packages()
else:
package_list = None
date = datetime.datetime.utcnow()
- commit = subprocess.check_output(['git', 'rev-parse',
+ commit = subprocess.check_output(['git', '-C', brpath,
+ 'rev-parse',
'HEAD']).splitlines()[0].decode()
print("Build package list ...")
packages = get_pkglist(args.npackages, package_list)
print("Getting developers ...")
- developers = parse_developers()
+ developers = parse_developers(brpath)
print("Build defconfig list ...")
defconfigs = get_defconfig_list()
for d in defconfigs:
@@ -965,6 +1089,7 @@ def __main__():
pkg.set_patch_count()
pkg.set_check_package_warnings()
pkg.set_current_version()
+ pkg.set_cpeid()
pkg.set_url()
pkg.set_developers(developers)
print("Checking URL status")
@@ -975,7 +1100,7 @@ def __main__():
loop.run_until_complete(check_package_latest_version(packages))
if args.nvd_path:
print("Checking packages CVEs")
- check_package_cves(args.nvd_path, {p.name: p for p in packages})
+ check_package_cves(args.nvd_path, packages)
print("Calculate stats")
stats = calculate_stats(packages)
if args.html: