support/scripts/gen-missing-cpe: remove rarely used script

The intention of this script is to generate the XML that can be sent to
NVD to request a new CPE identifier.

As discussed on the mailing list [0] keeping up with version numbers of
all registered CPE ID won't work.
In addition the feed used to generated the XML files will be retired
[1]. In the future an API needs to be used for fetching the data in
connection with a local database.
All of this works against keeping this script and porting it to the new
API.
As a last blow Matthew, the original author concluded [2]:
> Makes sense to drop it.  There never got to be enough momentum in the overall
> software community to make CVE or even the new identifier really accurate.

The intention is to ignore the version part of CPE IDs in the future,
and only look at the version range specified on a CVE. Therefore, a tool
to add new CPE ID versions isn't useful to us. It might still be useful
to have a tool to create the vendor and project parts of a CPE ID.
However, the current gen-missing-cpe tool doesn't support that, and the
API is anyway going to be retired. So there is no reason at all to keep
this around.

Remove gen-missing-cpe and the cpedb module. Remove the Makefile target
to call the script.

Since the cpedb module is removed, the CPEDB_URL definition must be
moved to the place where it is still used, in pkg-stats.

[0]: https://lists.buildroot.org/pipermail/buildroot/2023-August/672620.html
[1]: https://nvd.nist.gov/General/News/change-timeline
[2]: https://lists.buildroot.org/pipermail/buildroot/2023-August/672651.html

Signed-off-by: Daniel Lang <dalang@gmx.at>
Signed-off-by: Arnout Vandecappelle <arnout@mind.be>

Makefile

@@ -927,14 +927,6 @@ pkg-stats:
 		--html $(O)/pkg-stats.html \
 		--nvd-path $(DL_DIR)/buildroot-nvd
 
-.PHONY: missing-cpe
-missing-cpe:
-	$(Q)mkdir -p $(CPE_UPDATES_DIR)
-	$(Q)cd "$(CONFIG_DIR)" ; \
-	$(TOPDIR)/support/scripts/gen-missing-cpe \
-		--nvd-path $(DL_DIR)/buildroot-nvd \
-		--output $(CPE_UPDATES_DIR)
-
 else # ifeq ($(BR2_HAVE_DOT_CONFIG),y)
 
 # Some subdirectories are also package names. To avoid that "make linux"
@@ -1191,7 +1183,6 @@ help:
 	@echo '  legal-info             - generate info about license compliance'
 	@echo '  show-info              - generate info about packages, as a JSON blurb'
 	@echo '  pkg-stats              - generate info about packages as JSON and HTML'
-	@echo '  missing-cpe            - generate XML snippets for missing CPE identifiers'
 	@echo '  printvars              - dump internal variables selected with VARS=...'
 	@echo '  show-vars              - dump all internal variables as a JSON blurb; use VARS=...'
 	@echo '                           to limit the list to variables names matching that pattern'

support/scripts/cpedb.py

@@ -1,174 +0,0 @@
-#!/usr/bin/env python3
-
-import xml.etree.ElementTree as ET
-from xml.etree.ElementTree import Element, SubElement
-import gzip
-import os
-import requests
-import time
-from xml.dom import minidom
-
-VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
-
-CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
-
-ns = {
-    '': 'http://cpe.mitre.org/dictionary/2.0',
-    'cpe-23': 'http://scap.nist.gov/schema/cpe-extension/2.3',
-    'xml': 'http://www.w3.org/XML/1998/namespace'
-}
-
-
-class CPE:
-    def __init__(self, cpe_str, titles, refs):
-        self.cpe_str = cpe_str
-        self.titles = titles
-        self.references = refs
-        self.cpe_cur_ver = "".join(self.cpe_str.split(":")[5:6])
-
-    def update_xml_dict(self):
-        ET.register_namespace('', 'http://cpe.mitre.org/dictionary/2.0')
-        cpes = Element('cpe-list')
-        cpes.set('xmlns:cpe-23', "http://scap.nist.gov/schema/cpe-extension/2.3")
-        cpes.set('xmlns:ns6', "http://scap.nist.gov/schema/scap-core/0.1")
-        cpes.set('xmlns:scap-core', "http://scap.nist.gov/schema/scap-core/0.3")
-        cpes.set('xmlns:config', "http://scap.nist.gov/schema/configuration/0.1")
-        cpes.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance")
-        cpes.set('xmlns:meta', "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2")
-        cpes.set('xsi:schemaLocation', " ".join(["http://scap.nist.gov/schema/cpe-extension/2.3",
-                                                 "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd",
-                                                 "http://cpe.mitre.org/dictionary/2.0",
-                                                 "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd",
-                                                 "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
-                                                 "https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd",
-                                                 "http://scap.nist.gov/schema/scap-core/0.3",
-                                                 "https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd",
-                                                 "http://scap.nist.gov/schema/configuration/0.1",
-                                                 "https://scap.nist.gov/schema/nvd/configuration_0.1.xsd",
-                                                 "http://scap.nist.gov/schema/scap-core/0.1",
-                                                 "https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"]))
-        item = SubElement(cpes, 'cpe-item')
-        cpe_short_name = CPE.short_name(self.cpe_str)
-        cpe_new_ver = CPE.version_update(self.cpe_str)
-
-        item.set('name', 'cpe:/' + cpe_short_name)
-        self.titles[0].text.replace(self.cpe_cur_ver, cpe_new_ver)
-        for title in self.titles:
-            item.append(title)
-        if self.references:
-            item.append(self.references)
-        cpe23item = SubElement(item, 'cpe-23:cpe23-item')
-        cpe23item.set('name', self.cpe_str)
-
-        # Generate the XML as a string
-        xmlstr = ET.tostring(cpes)
-
-        # And use minidom to pretty print the XML
-        return minidom.parseString(xmlstr).toprettyxml(encoding="utf-8").decode("utf-8")
-
-    @staticmethod
-    def version(cpe):
-        return cpe.split(":")[5]
-
-    @staticmethod
-    def product(cpe):
-        return cpe.split(":")[4]
-
-    @staticmethod
-    def short_name(cpe):
-        return ":".join(cpe.split(":")[2:6])
-
-    @staticmethod
-    def version_update(cpe):
-        return ":".join(cpe.split(":")[5:6])
-
-    @staticmethod
-    def no_version(cpe):
-        return ":".join(cpe.split(":")[:5])
-
-
-class CPEDB:
-    def __init__(self, nvd_path):
-        self.all_cpes = dict()
-        self.all_cpes_no_version = dict()
-        self.nvd_path = nvd_path
-
-    def get_xml_dict(self):
-        print("CPE: Setting up NIST dictionary")
-        if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
-            os.makedirs(os.path.join(self.nvd_path, "cpe"))
-
-        cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
-        if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
-            print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
-            cpe_dict = requests.get(CPEDB_URL)
-            open(cpe_dict_local, "wb").write(cpe_dict.content)
-
-        print("CPE: Unzipping xml manifest...")
-        nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb'))
-        print("CPE: Converting xml manifest to dict...")
-        tree = ET.parse(nist_cpe_file)
-        all_cpedb = tree.getroot()
-        self.parse_dict(all_cpedb)
-
-    def parse_dict(self, all_cpedb):
-        # Cycle through the dict and build two dict to be used for custom
-        # lookups of partial and complete CPE objects
-        # The objects are then used to create new proposed XML updates if
-        # if is determined one is required
-        # Out of the different language titles, select English
-        for cpe in all_cpedb.findall(".//{http://cpe.mitre.org/dictionary/2.0}cpe-item"):
-            cpe_titles = []
-            for title in cpe.findall('.//{http://cpe.mitre.org/dictionary/2.0}title[@xml:lang="en-US"]', ns):
-                title.tail = None
-                cpe_titles.append(title)
-
-            # Some older CPE don't include references, if they do, make
-            # sure we handle the case of one ref needing to be packed
-            # in a list
-            cpe_ref = cpe.find(".//{http://cpe.mitre.org/dictionary/2.0}references")
-            if cpe_ref:
-                for ref in cpe_ref.findall(".//{http://cpe.mitre.org/dictionary/2.0}reference"):
-                    ref.tail = None
-                    ref.text = ref.text.upper()
-                    if ref.text not in VALID_REFS:
-                        ref.text = ref.text + "-- UPDATE this entry, here are some examples and just one word should be used -- " + ' '.join(VALID_REFS) # noqa E501
-                cpe_ref.tail = None
-                cpe_ref.text = None
-
-            cpe_str = cpe.find(".//{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item").get('name')
-            item = CPE(cpe_str, cpe_titles, cpe_ref)
-            cpe_str_no_version = CPE.no_version(cpe_str)
-            # This dict must have a unique key for every CPE version
-            # which allows matching to the specific obj data of that
-            # NIST dict entry
-            self.all_cpes.update({cpe_str: item})
-            # This dict has one entry for every CPE (w/o version) to allow
-            # partial match (no valid version) check (the obj is saved and
-            # used as seed for suggested xml updates. By updating the same
-            # non-version'd entry, it assumes the last update here is the
-            # latest version in the NIST dict)
-            self.all_cpes_no_version.update({cpe_str_no_version: item})
-
-    def find_partial(self, cpe_str):
-        cpe_str_no_version = CPE.no_version(cpe_str)
-        if cpe_str_no_version in self.all_cpes_no_version:
-            return cpe_str_no_version
-
-    def find_partial_obj(self, cpe_str):
-        cpe_str_no_version = CPE.no_version(cpe_str)
-        if cpe_str_no_version in self.all_cpes_no_version:
-            return self.all_cpes_no_version[cpe_str_no_version]
-
-    def find_partial_latest_version(self, cpe_str_partial):
-        cpe_obj = self.find_partial_obj(cpe_str_partial)
-        return cpe_obj.cpe_cur_ver
-
-    def find(self, cpe_str):
-        if self.find_partial(cpe_str):
-            if cpe_str in self.all_cpes:
-                return cpe_str
-
-    def gen_update_xml(self, cpe_str):
-        cpe = self.find_partial_obj(cpe_str)
-        return cpe.update_xml_dict()

support/scripts/gen-missing-cpe

@@ -1,65 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import sys
-import json
-import subprocess
-import os
-from cpedb import CPEDB, CPE
-
-
-def gen_update_xml_reports(cpeids, cpedb, output):
-    cpe_need_update = []
-
-    for cpe in cpeids:
-        result = cpedb.find(cpe)
-        if not result:
-            result = cpedb.find_partial(CPE.no_version(cpe))
-            if result:
-                cpe_need_update.append(cpe)
-            else:
-                print("WARNING: no match found for '%s'" % cpe)
-
-    for cpe in cpe_need_update:
-        xml = cpedb.gen_update_xml(cpe)
-        fname = CPE.product(cpe) + '-' + CPE.version(cpe) + '.xml'
-        print("Generating %s" % fname)
-        with open(os.path.join(output, fname), 'w+') as fp:
-            fp.write(xml)
-
-    print("Generated %d update files out of %d CPEs" % (len(cpe_need_update), len(cpeids)))
-
-
-def get_cpe_ids():
-    print("Getting list of CPE for enabled packages")
-    cmd = ["make", "--no-print-directory", "show-info"]
-    js = json.loads(subprocess.check_output(cmd).decode("utf-8"))
-    return set([v["cpe-id"] for k, v in js.items() if "cpe-id" in v])
-
-
-def resolvepath(path):
-    return os.path.abspath(os.path.expanduser(path))
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--output', dest='output',
-                        help='Path to the output CPE update files', type=resolvepath, required=True)
-    parser.add_argument('--nvd-path', dest='nvd_path',
-                        help='Path to the local NVD database', type=resolvepath, required=True)
-    return parser.parse_args()
-
-
-def __main__():
-    args = parse_args()
-    if not os.path.isdir(args.output):
-        print("ERROR: output directory %s does not exist" % args.output)
-        sys.exit(1)
-    cpedb = CPEDB(args.nvd_path)
-    cpedb.get_xml_dict()
-    cpeids = get_cpe_ids()
-    gen_update_xml_reports(cpeids, cpedb, args.output)
-
-
-if __name__ == "__main__":
-    __main__()

support/scripts/pkg-stats

@@ -37,10 +37,10 @@ brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
 
 sys.path.append(os.path.join(brpath, "utils"))
 from getdeveloperlib import parse_developers  # noqa: E402
-from cpedb import CPEDB_URL  # noqa: E402
 
 INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
 URL_RE = re.compile(r"\s*https?://\S*\s*$")
+CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
 
 RM_API_STATUS_ERROR = 1
 RM_API_STATUS_FOUND_BY_DISTRO = 2