support/scripts/pkg-stats: remove checking the CPE database

We currently check the CPE database to see if the CPE ID we use
(including the version) is already in the database.

However, the version part of the CPE ID is not actually used for CVE
matching. Instead, the CVEs have a range of versions associated with
them and we match against those ranges.

In addition, NVD is moving to a new API for accessing the CPE database.
It will not longer be possible to simply download all the CPE IDs, and
due to rate limiting, the download will have to be done in several
queries.

Since all of this is anyway of limited use, drop the CPE database lookup
entirely. Instead, as long as a CPE ID is defined in a package, it is
considered OK, without any checks.

Signed-off-by: Arnout Vandecappelle <arnout@mind.be>
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com>

support/scripts/pkg-stats

@@ -28,10 +28,6 @@ import re
 import subprocess
 import json
 import sys
-import time
-import gzip
-import xml.etree.ElementTree
-import requests
 
 brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
 
@@ -40,7 +36,6 @@ from getdeveloperlib import parse_developers  # noqa: E402
 
 INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
 URL_RE = re.compile(r"\s*https?://\S*\s*$")
-CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
 
 RM_API_STATUS_ERROR = 1
 RM_API_STATUS_FOUND_BY_DISTRO = 2
@@ -282,8 +277,7 @@ class Package:
 
         if var in self.all_cpeids:
             self.cpeid = self.all_cpeids[var]
-            # Set a preliminary status, it might be overridden by check_package_cpes()
-            self.status['cpe'] = ("warning", "not checked against CPE dictionary")
+            self.status['cpe'] = ("ok", "(not checked against CPE dictionary)")
         else:
             self.status['cpe'] = ("error", "no verified CPE identifier")
 
@@ -674,47 +668,6 @@ def check_package_cves(nvd_path, packages):
                 pkg.status['cve'] = ("ok", "not affected by CVEs")
 
 
-def check_package_cpes(nvd_path, packages):
-    class CpeXmlParser:
-        cpes = []
-
-        def start(self, tag, attrib):
-            if tag == "{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item":
-                self.cpes.append(attrib['name'])
-
-        def close(self):
-            return self.cpes
-
-    print("CPE: Setting up NIST dictionary")
-    if not os.path.exists(os.path.join(nvd_path, "cpe")):
-        os.makedirs(os.path.join(nvd_path, "cpe"))
-
-    cpe_dict_local = os.path.join(nvd_path, "cpe", os.path.basename(CPEDB_URL))
-    if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
-        print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
-        cpe_dict = requests.get(CPEDB_URL)
-        open(cpe_dict_local, "wb").write(cpe_dict.content)
-
-    print("CPE: Unzipping xml manifest...")
-    nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb'))
-
-    parser = xml.etree.ElementTree.XMLParser(target=CpeXmlParser())
-    while True:
-        c = nist_cpe_file.read(1024*1024)
-        if not c:
-            break
-        parser.feed(c)
-    cpes = parser.close()
-
-    for p in packages:
-        if not p.cpeid:
-            continue
-        if p.cpeid in cpes:
-            p.status['cpe'] = ("ok", "verified CPE identifier")
-        else:
-            p.status['cpe'] = ("error", "CPE version unknown in CPE database")
-
-
 def calculate_stats(packages):
     stats = defaultdict(int)
     stats['packages'] = len(packages)
@@ -1309,7 +1262,7 @@ def parse_args():
     parser.add_argument('--nvd-path', dest='nvd_path',
                         help='Path to the local NVD database', type=resolvepath)
     parser.add_argument('--disable', type=list_str,
-                        help='Features to disable, comma-separated (cve, upstream, url, cpe, warning)',
+                        help='Features to disable, comma-separated (cve, upstream, url, warning)',
                         default=[])
     args = parser.parse_args()
     if not args.html and not args.json:
@@ -1372,9 +1325,6 @@ def __main__():
     if "cve" not in args.disable and args.nvd_path:
         print("Checking packages CVEs")
         check_package_cves(args.nvd_path, packages)
-    if "cpe" not in args.disable and args.nvd_path:
-        print("Checking packages CPEs")
-        check_package_cpes(args.nvd_path, packages)
     print("Calculate stats")
     stats = calculate_stats(packages)
     if args.html: