bitbake: bitbake: data_smart.py and friends: Track file inclusions for bitbake -e

This code adds inclusion history to bitbake -e output, showing
which files were included, in what order. This doesn't completely
resolve timing questions, because it doesn't show you which lines
of a file were processed before or after a given include, but it
does let you figure out what the path was by which a particular
file ended up in your build at all.

How it works: data_smart acquires a .history member, which is an
IncludeHistory; this represents the inclusion of a file and all its
inclusions, recursively. It provides methods for including files,
for finishing inclusion (done as an __exit__), and for
dumping the whole tree.

The parser is modified to run includes inside a with() to push
and pop the include filename.

RP Modifications:

a) Split Include and Variable tracking
b) Replace deepcopy usage with dedicated copy function
c) Simplify some variable and usage

(Bitbake rev: b2dda721262da8abb7dc32d019e18fbc32ed8860)

Signed-off-by: Peter Seebach <peter.seebach@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cooker.py

@@ -338,6 +338,11 @@ class BBCooker:
                 parselog.exception("Unable to read %s", fn)
                 raise
 
+        # Display history
+        with closing(StringIO()) as env:
+            self.configuration.data.inchistory.emit(env)
+            logger.plain(env.getvalue())
+
         # emit variables and shell functions
         data.update_data(envdata)
         with closing(StringIO()) as env:

bitbake/lib/bb/data_smart.py

@@ -114,10 +114,55 @@ class ExpansionError(Exception):
     def __str__(self):
         return self.msg
 
+class IncludeHistory(object):
+    def __init__(self, parent = None, filename = '[TOP LEVEL]'):
+        self.parent = parent
+        self.filename = filename
+        self.children = []
+        self.current = self
+
+    def copy(self):
+        new = IncludeHistory(self.parent, self.filename)
+        for c in self.children:
+            new.children.append(c)
+        return new
+
+    def include(self, filename):
+        newfile = IncludeHistory(self.current, filename)
+        self.current.children.append(newfile)
+        self.current = newfile
+        return self
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, a, b, c):
+        if self.current.parent:
+            self.current = self.current.parent
+        else:
+            bb.warn("Include log: Tried to finish '%s' at top level." % filename)
+        return False
+
+    def emit(self, o, level = 0):
+        """Emit an include history file, and its children."""
+        if level:
+            spaces = "  " * (level - 1)
+            o.write("# %s%s" % (spaces, self.filename))
+            if len(self.children) > 0:
+                o.write(" includes:")
+        else:
+            o.write("#\n# INCLUDE HISTORY:\n#")
+        level = level + 1
+        for child in self.children:
+            o.write("\n")
+            child.emit(o, level)
+
 class DataSmart(MutableMapping):
     def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ):
         self.dict = {}
 
+        self.inchistory = IncludeHistory()
+
         # cookie monster tribute
         self._special_values = special
         self._seen_overrides = seen
@@ -416,6 +461,7 @@ class DataSmart(MutableMapping):
         # we really want this to be a DataSmart...
         data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy())
         data.dict["_data"] = self.dict
+        data.inchistory = self.inchistory.copy()
 
         return data
 

bitbake/lib/bb/parse/__init__.py

@@ -87,7 +87,8 @@ def handle(fn, data, include = 0):
     """Call the handler that is appropriate for this file"""
     for h in handlers:
         if h['supports'](fn, data):
-            return h['handle'](fn, data, include)
+            with data.inchistory.include(fn):
+                return h['handle'](fn, data, include)
     raise ParseError("not a BitBake file", fn)
 
 def init(fn, data):