mirror of
https://git.yoctoproject.org/poky
synced 2026-05-09 05:29:32 +00:00
cve-update-db-native: add the fkie source
Add support for FKIE-CAD reconstruction of NVD feed from https://github.com/fkie-cad/nvd-json-data-feeds We download this feed directly from github releases. (From OE-Core rev: f6253ac8189db09fbe87141aca1733cb37a4d78f) Signed-off-by: Marta Rybczynska <marta.rybczynska@ygreky.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Marta Rybczynska
committed by
Richard Purdie
Richard Purdie
parent
7a3904c6a7
commit
0486af6e3c
@@ -12,6 +12,8 @@ deltask do_install
|
||||
deltask do_populate_sysroot
|
||||
|
||||
NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-"
|
||||
FKIE_URL ?= "https://github.com/fkie-cad/nvd-json-data-feeds/releases/latest/download/CVE-"
|
||||
|
||||
# CVE database update interval, in seconds. By default: once a day (24*60*60).
|
||||
# Use 0 to force the update
|
||||
# Use a negative value to skip the update
|
||||
@@ -109,6 +111,30 @@ def cleanup_db_download(db_file, db_tmp_file):
|
||||
if os.path.exists(db_tmp_file):
|
||||
os.remove(db_tmp_file)
|
||||
|
||||
def db_file_names(d, year, is_nvd):
|
||||
if is_nvd:
|
||||
year_url = d.getVar('NVDCVE_URL') + str(year)
|
||||
meta_url = year_url + ".meta"
|
||||
json_url = year_url + ".json.gz"
|
||||
return json_url, meta_url
|
||||
year_url = d.getVar('FKIE_URL') + str(year)
|
||||
meta_url = year_url + ".meta"
|
||||
json_url = year_url + ".json.xz"
|
||||
return json_url, meta_url
|
||||
|
||||
def host_db_name(d, is_nvd):
|
||||
if is_nvd:
|
||||
return "nvd.nist.gov"
|
||||
return "github.com"
|
||||
|
||||
def db_decompress(d, data, is_nvd):
|
||||
import gzip, lzma
|
||||
|
||||
if is_nvd:
|
||||
return gzip.decompress(data).decode('utf-8')
|
||||
# otherwise
|
||||
return lzma.decompress(data)
|
||||
|
||||
def update_db_file(db_tmp_file, d):
|
||||
"""
|
||||
Update the given database file
|
||||
@@ -119,6 +145,7 @@ def update_db_file(db_tmp_file, d):
|
||||
|
||||
YEAR_START = 2002
|
||||
cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
|
||||
is_nvd = d.getVar("NVD_DB_VERSION") == "NVD1"
|
||||
|
||||
# Connect to database
|
||||
conn = sqlite3.connect(db_tmp_file)
|
||||
@@ -129,9 +156,7 @@ def update_db_file(db_tmp_file, d):
|
||||
for i, year in enumerate(range(YEAR_START, date.today().year + 1)):
|
||||
bb.debug(2, "Updating %d" % year)
|
||||
ph.update((float(i + 1) / total_years) * 100)
|
||||
year_url = (d.getVar('NVDCVE_URL')) + str(year)
|
||||
meta_url = year_url + ".meta"
|
||||
json_url = year_url + ".json.gz"
|
||||
json_url, meta_url = db_file_names(d, year, is_nvd)
|
||||
|
||||
# Retrieve meta last modified date
|
||||
try:
|
||||
@@ -140,7 +165,7 @@ def update_db_file(db_tmp_file, d):
|
||||
cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n')
|
||||
bb.warn("Failed to fetch CVE data (%s)" % e)
|
||||
import socket
|
||||
result = socket.getaddrinfo("nvd.nist.gov", 443, proto=socket.IPPROTO_TCP)
|
||||
result = socket.getaddrinfo(host_db_name(d, is_nvd), 443, proto=socket.IPPROTO_TCP)
|
||||
bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t in result)))
|
||||
return False
|
||||
|
||||
@@ -168,7 +193,7 @@ def update_db_file(db_tmp_file, d):
|
||||
try:
|
||||
response = urllib.request.urlopen(json_url, timeout=cve_socket_timeout)
|
||||
if response:
|
||||
update_db(conn, gzip.decompress(response.read()).decode('utf-8'))
|
||||
update_db(d, conn, db_decompress(d, response.read(), is_nvd))
|
||||
conn.execute("insert or replace into META values (?, ?)", [year, last_modified]).close()
|
||||
except urllib.error.URLError as e:
|
||||
cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
|
||||
@@ -200,16 +225,22 @@ def initialize_db(conn):
|
||||
|
||||
c.close()
|
||||
|
||||
def parse_node_and_insert(conn, node, cveId):
|
||||
def parse_node_and_insert(conn, node, cveId, is_nvd):
|
||||
# Parse children node if needed
|
||||
for child in node.get('children', ()):
|
||||
parse_node_and_insert(conn, child, cveId)
|
||||
parse_node_and_insert(conn, child, cveId, is_nvd)
|
||||
|
||||
def cpe_generator():
|
||||
for cpe in node.get('cpe_match', ()):
|
||||
def cpe_generator(is_nvd):
|
||||
match_string = "cpeMatch"
|
||||
cpe_string = 'criteria'
|
||||
if is_nvd:
|
||||
match_string = "cpe_match"
|
||||
cpe_string = 'cpe23Uri'
|
||||
|
||||
for cpe in node.get(match_string, ()):
|
||||
if not cpe['vulnerable']:
|
||||
return
|
||||
cpe23 = cpe.get('cpe23Uri')
|
||||
cpe23 = cpe.get(cpe_string)
|
||||
if not cpe23:
|
||||
return
|
||||
cpe23 = cpe23.split(':')
|
||||
@@ -260,9 +291,9 @@ def parse_node_and_insert(conn, node, cveId):
|
||||
# Save processing by representing as -.
|
||||
yield [cveId, vendor, product, '-', '', '', '']
|
||||
|
||||
conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close()
|
||||
conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator(is_nvd)).close()
|
||||
|
||||
def update_db(conn, jsondata):
|
||||
def update_db_nvdjson(conn, jsondata):
|
||||
import json
|
||||
root = json.loads(jsondata)
|
||||
|
||||
@@ -297,8 +328,77 @@ def update_db(conn, jsondata):
|
||||
|
||||
configurations = elt['configurations']['nodes']
|
||||
for config in configurations:
|
||||
parse_node_and_insert(conn, config, cveId)
|
||||
parse_node_and_insert(conn, config, cveId, True)
|
||||
|
||||
def update_db_fkie(conn, jsondata):
|
||||
import json
|
||||
root = json.loads(jsondata)
|
||||
|
||||
for elt in root['cve_items']:
|
||||
if not 'vulnStatus' in elt or elt['vulnStatus'] == 'Rejected':
|
||||
continue
|
||||
|
||||
if not 'configurations' in elt:
|
||||
continue
|
||||
|
||||
accessVector = None
|
||||
vectorString = None
|
||||
cvssv2 = 0.0
|
||||
cvssv3 = 0.0
|
||||
cvssv4 = 0.0
|
||||
cveId = elt['id']
|
||||
cveDesc = elt['descriptions'][0]['value']
|
||||
date = elt['lastModified']
|
||||
try:
|
||||
for m in elt['metrics']['cvssMetricV2']:
|
||||
if m['type'] == 'Primary':
|
||||
accessVector = m['cvssData']['accessVector']
|
||||
vectorString = m['cvssData']['vectorString']
|
||||
cvssv2 = m['cvssData']['baseScore']
|
||||
except KeyError:
|
||||
cvssv2 = 0.0
|
||||
try:
|
||||
for m in elt['metrics']['cvssMetricV30']:
|
||||
if m['type'] == 'Primary':
|
||||
accessVector = m['cvssData']['accessVector']
|
||||
vectorString = m['cvssData']['vectorString']
|
||||
cvssv3 = m['cvssData']['baseScore']
|
||||
except KeyError:
|
||||
accessVector = accessVector or "UNKNOWN"
|
||||
cvssv3 = 0.0
|
||||
try:
|
||||
for m in elt['metrics']['cvssMetricV31']:
|
||||
if m['type'] == 'Primary':
|
||||
accessVector = m['cvssData']['accessVector']
|
||||
vectorString = m['cvssData']['vectorString']
|
||||
cvssv3 = m['cvssData']['baseScore']
|
||||
except KeyError:
|
||||
accessVector = accessVector or "UNKNOWN"
|
||||
cvssv3 = 0.0
|
||||
try:
|
||||
for m in elt['metrics']['cvssMetricV40']:
|
||||
if m['type'] == 'Primary':
|
||||
accessVector = m['cvssData']['accessVector']
|
||||
vectorString = m['cvssData']['vectorString']
|
||||
cvssv4 = m['cvssData']['baseScore']
|
||||
except KeyError:
|
||||
accessVector = accessVector or "UNKNOWN"
|
||||
cvssv4 = 0.0
|
||||
|
||||
conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
[cveId, cveDesc, cvssv2, cvssv3, cvssv4, date, accessVector, vectorString]).close()
|
||||
|
||||
for config in elt['configurations']:
|
||||
# This is suboptimal as it doesn't handle AND/OR and negate, but is better than nothing
|
||||
for node in config["nodes"]:
|
||||
parse_node_and_insert(conn, node, cveId, False)
|
||||
|
||||
|
||||
def update_db(d, conn, jsondata):
|
||||
if (d.getVar("NVD_DB_VERSION") == "FKIE"):
|
||||
return update_db_fkie(conn, jsondata)
|
||||
else:
|
||||
return update_db_nvdjson(conn, jsondata)
|
||||
|
||||
do_fetch[nostamp] = "1"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user