From 54e4a89a758906102067dca19f1d3212e0d76ad9 Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Thu, 13 Feb 2025 10:18:17 -0700 Subject: [PATCH] spdx30: Improve os.walk() handling There have been errors seen when assembling root file system SPDX documents where they will references files that don't exist in the package SPDX. The speculation is that this is caused by os.walk() ignoring errors when walking, causing files to be omitted. Improve the code by adding an error handler to os.walk() to report errors when they occur. In addition, sort the files and directories while walking to ensure consistent ordering of the file SPDX IDs. (From OE-Core rev: 86b581e80637cd8136ce7a7e95db94d9553d2f60) Signed-off-by: Joshua Watt Signed-off-by: Richard Purdie --- meta/lib/oe/spdx30_tasks.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py index 6a39246fe1..e3e5dbc742 100644 --- a/meta/lib/oe/spdx30_tasks.py +++ b/meta/lib/oe/spdx30_tasks.py @@ -19,6 +19,10 @@ from datetime import datetime, timezone from pathlib import Path +def walk_error(err): + bb.error(f"ERROR walking {err.filename}: {err}") + + def set_timestamp_now(d, o, prop): if d.getVar("SPDX_INCLUDE_TIMESTAMPS") == "1": setattr(o, prop, datetime.now(timezone.utc)) @@ -148,11 +152,13 @@ def add_package_files( spdx_files = set() file_counter = 1 - for subdir, dirs, files in os.walk(topdir): + for subdir, dirs, files in os.walk(topdir, onerror=walk_error): dirs[:] = [d for d in dirs if d not in ignore_dirs] if subdir == str(topdir): dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] + dirs.sort() + files.sort() for file in files: filepath = Path(subdir) / file if filepath.is_symlink() or not filepath.is_file(): @@ -356,7 +362,9 @@ def add_download_files(d, objset): if fd.type == "file": if os.path.isdir(fd.localpath): walk_idx = 1 - for root, dirs, files in os.walk(fd.localpath): + for root, dirs, files in os.walk(fd.localpath, onerror=walk_error): + dirs.sort() + files.sort() for f in files: f_path = os.path.join(root, f) if os.path.islink(f_path): @@ -1046,7 +1054,9 @@ def create_rootfs_spdx(d): collect_build_package_inputs(d, objset, rootfs_build, packages, files_by_hash) files = set() - for dirpath, dirnames, filenames in os.walk(image_rootfs): + for dirpath, dirnames, filenames in os.walk(image_rootfs, onerror=walk_error): + dirnames.sort() + filenames.sort() for fn in filenames: fpath = Path(dirpath) / fn if not fpath.is_file() or fpath.is_symlink(): @@ -1282,7 +1292,9 @@ def create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, toolchain_outputname): root_files = [] # NOTE: os.walk() doesn't return symlinks - for dirpath, dirnames, filenames in os.walk(sdk_deploydir): + for dirpath, dirnames, filenames in os.walk(sdk_deploydir, onerror=walk_error): + dirnames.sort() + filenames.sort() for fn in filenames: fpath = Path(dirpath) / fn if not fpath.is_file() or fpath.is_symlink():