build: use bb.process instead of os.system

(Bitbake rev: 53740977521bc81ffa37adfa7bbeb8f2a80ea165)

build: write logfiles per task, not per function
Based on d14f9bf6 from poky, reworked for master and other cleanup.

(Bitbake rev: beadff2eca1eb95f0411115dd72ddb4c3c44c604)

Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

bitbake/lib/bb/build.py

@@ -31,9 +31,13 @@ import sys
 import logging
 import bb
 import bb.utils
+import bb.process
 
 logger = logging.getLogger("BitBake.Build")
 
+NULL = open('/dev/null', 'r')
+
+
 # When we execute a python function we'd like certain things
 # in all namespaces, hence we add them to __builtins__
 # If we do not do this and use the exec globals, they will
@@ -53,8 +57,8 @@ class FuncFailed(Exception):
 
     def __str__(self):
         if self.logfile and os.path.exists(self.logfile):
-            msg = "%s (see %s for further information)" % \
+            msg = ("%s (see %s for further information)" %
-                  (self.message, self.logfile)
+                   (self.message, self.logfile))
         else:
             msg = self.message
         return msg
@@ -95,30 +99,33 @@ class TaskInvalid(TaskBase):
         super(TaskInvalid, self).__init__(task, metadata)
         self._message = "No such task '%s'" % task
 
-# functions
 
-def exec_func(func, d, dirs = None):
+class tee(file):
+    def write(self, string):
+        logger.plain(string)
+        file.write(self, string)
+
+    def __repr__(self):
+        return "<open[tee] file '{0}'>".format(self.name)
+
+
+def exec_func(func, d, dirs = None, logfile = NULL):
     """Execute an BB 'function'"""
 
     body = data.getVar(func, d)
     if not body:
-        bb.warn("Function %s doesn't exist" % func)
+        if body is None:
+            logger.warn("Function %s doesn't exist", func)
         return
 
     flags = data.getVarFlags(func, d)
-    for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot', 'task']:
+    cleandirs = flags.get('cleandirs')
-        if not item in flags:
-            flags[item] = None
-
-    ispython = flags['python']
-
-    cleandirs = flags['cleandirs']
     if cleandirs:
         for cdir in data.expand(cleandirs, d).split():
-            os.system("rm -rf %s" % cdir)
+            bb.utils.remove(cdir, True)
 
     if dirs is None:
-        dirs = flags['dirs']
+        dirs = flags.get('dirs')
         if dirs:
             dirs = data.expand(dirs, d).split()
 
@@ -128,214 +135,185 @@ def exec_func(func, d, dirs = None):
         adir = dirs[-1]
     else:
         adir = data.getVar('B', d, 1)
+        if not os.path.exists(adir):
+            adir = None
 
-    # Save current directory
+    ispython = flags.get('python')
-    try:
+    if flags.get('fakeroot') and not flags.get('task'):
-        prevdir = os.getcwd()
+        bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func)
-    except OSError:
-        prevdir = data.getVar('TOPDIR', d, True)
 
-    # Setup scriptfile
+    tempdir = data.getVar('T', d, 1)
-    t = data.getVar('T', d, 1)
+    runfile = os.path.join(tempdir, 'run.{0}.{1}'.format(func, os.getpid()))
-    if not t:
-        raise SystemExit("T variable not set, unable to build")
-    bb.utils.mkdirhier(t)
-    runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
-    logfile = d.getVar("BB_LOGFILE", True)
-
-    # Change to correct directory (if specified)
-    if adir and os.access(adir, os.F_OK):
-        os.chdir(adir)
 
     locks = []
-    lockfiles = flags['lockfiles']
+    lockfiles = flags.get('lockfiles')
     if lockfiles:
         for lock in data.expand(lockfiles, d).split():
             locks.append(bb.utils.lockfile(lock))
 
     try:
-        # Run the function
         if ispython:
-            exec_func_python(func, d, runfile, logfile)
+            exec_func_python(func, d, runfile, logfile, cwd=adir)
         else:
-            exec_func_shell(func, d, runfile, logfile, flags)
+            exec_func_shell(func, d, runfile, logfile, cwd=adir)
-
-        # Restore original directory
-        try:
-            os.chdir(prevdir)
-        except:
-            pass
 
     finally:
-
         # Unlock any lockfiles
         for lock in locks:
             bb.utils.unlockfile(lock)
 
-def exec_func_python(func, d, runfile, logfile):
+_functionfmt = """
+def {function}(d):
+{body}
+
+{function}(d)
+"""
+#logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+def exec_func_python(func, d, runfile, logfile, cwd=None):
     """Execute a python BB 'function'"""
 
-    bbfile = bb.data.getVar('FILE', d, 1)
+    bbfile = d.getVar('FILE', True)
-    tmp  = "def " + func + "(d):\n%s" % data.getVar(func, d)
-    tmp += '\n' + func + '(d)'
-
-    f = open(runfile, "w")
-    f.write(tmp)
-    comp = utils.better_compile(tmp, func, bbfile)
     try:
-        utils.better_exec(comp, {"d": d}, tmp, bbfile)
+        olddir = os.getcwd()
+    except OSError:
+        olddir = None
+    code = _functionfmt.format(function=func, body=d.getVar(func, True))
+    bb.utils.mkdirhier(os.path.dirname(runfile))
+    with open(runfile, 'w') as script:
+        script.write(code)
+
+    if cwd:
+        os.chdir(cwd)
+
+    #handler = logging.StreamHandler(logfile)
+    #handler.setFormatter(logformatter)
+    #bblogger.addHandler(handler)
+
+    try:
+        comp = utils.better_compile(code, func, bbfile)
+        utils.better_exec(comp, {"d": d}, code, bbfile)
     except:
         if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed):
             raise
 
-        raise FuncFailed(func, logfile)
+        raise FuncFailed(func, None)
+    finally:
+        #bblogger.removeHandler(handler)
+        if olddir:
+            os.chdir(olddir)
 
-
+def exec_func_shell(function, d, runfile, logfile, cwd=None):
-def exec_func_shell(func, d, runfile, logfile, flags):
+    """Execute a shell function from the metadata
-    """Execute a shell BB 'function' Returns true if execution was successful.
-
-    For this, it creates a bash shell script in the tmp dectory, writes the local
-    data into it and finally executes. The output of the shell will end in a log file and stdout.
 
     Note on directory behavior.  The 'dirs' varflag should contain a list
     of the directories you need created prior to execution.  The last
     item in the list is where we will chdir/cd to.
     """
 
-    deps = flags['deps']
+    # Don't let the emitted shell script override PWD
-    check = flags['check']
+    d.delVarFlag('PWD', 'export')
-    if check in globals():
+
-        if globals()[check](func, deps):
+    with open(runfile, 'w') as script:
-            return
+        script.write('#!/bin/sh -e\n')
+        if logger.getEffectiveLevel() <= logging.DEBUG:
+            script.write("set -x\n")
+        data.emit_func(function, script, d)
+
+        script.write("%s\n" % function)
+        os.fchmod(script.fileno(), 0775)
+
+    env = {
+        'PATH': d.getVar('PATH', True),
+        'LC_ALL': 'C',
+    }
+
+    cmd = runfile
 
-    f = open(runfile, "w")
-    f.write("#!/bin/sh -e\n")
     if logger.getEffectiveLevel() <= logging.DEBUG:
-        f.write("set -x\n")
+        logfile = LogTee(logger, logfile)
-    data.emit_func(func, f, d)
 
-    f.write("cd %s\n" % os.getcwd())
+    try:
-    if func: f.write("%s\n" % func)
+        bb.process.run(cmd, env=env, cwd=cwd, shell=False, stdin=NULL,
-    f.close()
+                       log=logfile)
-    os.chmod(runfile, 0775)
+    except bb.process.CmdError:
-    if not func:
+        raise FuncFailed(function, logfile.name)
-        raise TypeError("Function argument must be a string")
 
-    # execute function
+def _task_data(fn, task, d):
-    if flags['fakeroot'] and not flags['task']:
+    localdata = data.createCopy(d)
-        bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func)
+    localdata.setVar('BB_FILENAME', fn)
+    localdata.setVar('BB_CURRENTTASK', task[3:])
+    localdata.setVar('OVERRIDES', 'task-%s:%s' %
+                     (task[3:], d.getVar('OVERRIDES', False)))
+    localdata.finalize()
+    data.expandKeys(localdata)
+    return localdata
 
-    lang_environment = "LC_ALL=C "
+def _exec_task(fn, task, d, quieterr):
-    ret = os.system('%ssh -e %s' % (lang_environment, runfile))
+    """Execute a BB 'task'
 
-    if ret == 0:
+    Execution of a task involves a bit more setup than executing a function,
-        return
+    running it with its own local metadata, and with some useful variables set.
-
+    """
-    raise FuncFailed(func, logfile)
-
-
-def exec_task(fn, task, d):
-    """Execute an BB 'task'
-
-       The primary difference between executing a task versus executing
-       a function is that a task exists in the task digraph, and therefore
-       has dependencies amongst other tasks."""
-
-    # Check whther this is a valid task
     if not data.getVarFlag(task, 'task', d):
         event.fire(TaskInvalid(task, d), d)
         logger.error("No such task: %s" % task)
         return 1
 
-    quieterr = False
+    logger.debug(1, "Executing task %s", task)
-    if d.getVarFlag(task, "quieterrors") is not None:
-         quieterr = True
 
-    try:
+    localdata = _task_data(fn, task, d)
-        logger.debug(1, "Executing task %s", task)
+    tempdir = localdata.getVar('T', True)
-        old_overrides = data.getVar('OVERRIDES', d, 0)
+    if not tempdir:
-        localdata = data.createCopy(d)
+        bb.fatal("T variable not set, unable to build")
-        data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata)
-        data.update_data(localdata)
-        data.expandKeys(localdata)
-        data.setVar('BB_FILENAME', fn, d)
-        data.setVar('BB_CURRENTTASK', task[3:], d)
-        event.fire(TaskStarted(task, localdata), localdata)
 
-        # Setup logfiles
+    bb.utils.mkdirhier(tempdir)
-        t = data.getVar('T', d, 1)
+    loglink = os.path.join(tempdir, 'log.{0}'.format(task))
-        if not t:
+    logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
-            raise SystemExit("T variable not set, unable to build")
+    if loglink:
-        bb.utils.mkdirhier(t)
+        bb.utils.remove(loglink)
-        loglink = "%s/log.%s" % (t, task)
-        logfile = "%s/log.%s.%s" % (t, task, str(os.getpid()))
-        d.setVar("BB_LOGFILE", logfile)
 
-        # Even though the log file has not yet been opened, lets create the link
-        if loglink:
-            try:
-                os.remove(loglink)
-            except OSError as e:
-                pass
-
-            try:
-                os.symlink(logfile, loglink)
-            except OSError as e:
-                pass
-
-        # Handle logfiles
-        si = file('/dev/null', 'r')
         try:
-            so = file(logfile, 'w')
+           os.symlink(logfn, loglink)
         except OSError:
-            logger.exception("Opening log file '%s'", logfile)
+           pass
-            pass
-        se = so
 
-        # Dup the existing fds so we dont lose them
+    prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
-        osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
+    postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
-        oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
-        ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
 
-        # Replace those fds with our own
+    # Handle logfiles
-        os.dup2(si.fileno(), osi[1])
+    si = file('/dev/null', 'r')
-        os.dup2(so.fileno(), oso[1])
+    try:
-        os.dup2(se.fileno(), ose[1])
+        logfile = file(logfn, 'w')
+    except OSError:
+        logger.exception("Opening log file '%s'", logfn)
+        pass
 
-        # Since we've remapped stdout and stderr, its safe for log messages to be printed there now
+    # Dup the existing fds so we dont lose them
-        # exec_func can nest so we have to save state
+    osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
-        origstdout = bb.event.useStdout
+    oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
-        bb.event.useStdout = True
+    ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
 
+    # Replace those fds with our own
+    os.dup2(si.fileno(), osi[1])
+    os.dup2(logfile.fileno(), oso[1])
+    os.dup2(logfile.fileno(), ose[1])
 
-        prefuncs = (data.getVarFlag(task, 'prefuncs', localdata) or "").split()
+    # Since we've remapped stdout and stderr, its safe for log messages to be printed there now
-        for func in prefuncs:
+    # exec_func can nest so we have to save state
-            exec_func(func, localdata)
+    origstdout = bb.event.useStdout
-        exec_func(task, localdata)
+    bb.event.useStdout = True
-        postfuncs = (data.getVarFlag(task, 'postfuncs', localdata) or "").split()
-        for func in postfuncs:
-            exec_func(func, localdata)
-
-        event.fire(TaskSucceeded(task, localdata), localdata)
-
-        # make stamp, or cause event and raise exception
-        if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d):
-            make_stamp(task, d)
 
+    event.fire(TaskStarted(task, localdata), localdata)
+    try:
+        for func in (prefuncs or '').split():
+            exec_func(func, localdata, logfile=logfile)
+        exec_func(task, localdata, logfile=logfile)
+        for func in (postfuncs or '').split():
+            exec_func(func, localdata, logfile=logfile)
     except FuncFailed as exc:
         if not quieterr:
             logger.error(str(exc))
-            failedevent = TaskFailed(exc.name, exc.logfile, task, d)
+            event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata)
-            event.fire(failedevent, d)
-        return 1
-
-    except Exception:
-        from traceback import format_exc
-        if not quieterr:
-            logger.error("Build of %s failed" % (task))
-            logger.error(format_exc())
-            failedevent = TaskFailed("Task Failed", None, task, d)
-            event.fire(failedevent, d)
         return 1
     finally:
         sys.stdout.flush()
@@ -348,26 +326,40 @@ def exec_task(fn, task, d):
         os.dup2(oso[0], oso[1])
         os.dup2(ose[0], ose[1])
 
-        # Close our logs
-        si.close()
-        so.close()
-        se.close()
-
-        if logfile and os.path.exists(logfile) and os.path.getsize(logfile) == 0:
-            logger.debug(2, "Zero size logfile %s, removing", logfile)
-            os.remove(logfile)
-            try:
-               os.remove(loglink)
-            except OSError as e:
-               pass
-
         # Close the backup fds
         os.close(osi[0])
         os.close(oso[0])
         os.close(ose[0])
+        si.close()
+
+        logfile.close()
+        if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
+            logger.debug(2, "Zero size logfn %s, removing", logfn)
+            bb.utils.remove(logfn)
+            bb.utils.remove(loglink)
+    event.fire(TaskSucceeded(task, localdata), localdata)
+
+    if not d.getVarFlag(task, 'nostamp') and not d.getVarFlag(task, 'selfstamp'):
+        make_stamp(task, d)
 
     return 0
 
+def exec_task(fn, task, d):
+    try: 
+        quieterr = False
+        if d.getVarFlag(task, "quieterrors") is not None:
+            quieterr = True
+
+        return _exec_task(fn, task, d, quieterr)
+    except Exception:
+        from traceback import format_exc
+        if not quieterr:
+            logger.error("Build of %s failed" % (task))
+            logger.error(format_exc())
+            failedevent = TaskFailed("Task Failed", None, task, d)
+            event.fire(failedevent, d)
+        return 1
+
 def extract_stamp(d, fn):
     """
     Extracts stamp format which is either a data dictionary (fn unset)

bitbake/lib/bb/data_smart.py

@@ -291,13 +291,13 @@ class DataSmart(MutableMapping):
             self._makeShadowCopy(var)
         self.dict[var][flag] = flagvalue
 
-    def getVarFlag(self, var, flag, exp = False):
+    def getVarFlag(self, var, flag, expand = False):
         local_var = self._findVar(var)
         value = None
         if local_var:
             if flag in local_var:
                 value = copy.copy(local_var[flag])
-        if exp and value:
+        if expand and value:
             value = self.expand(value, None)
         return value
 

bitbake/lib/bb/utils.py

@@ -579,6 +579,17 @@ def build_environment(d):
         if export:
             os.environ[var] = bb.data.getVar(var, d, True) or ""
 
+def remove(path, recurse=False):
+    """Equivalent to rm -f or rm -rf"""
+    import os, errno, shutil
+    try:
+        os.unlink(path)
+    except OSError, exc:
+        if recurse and exc.errno == errno.EISDIR:
+            shutil.rmtree(path)
+        elif exc.errno != errno.ENOENT:
+            raise
+
 def prunedir(topdir):
     # Delete everything reachable from the directory named in 'topdir'.
     # CAUTION:  This is dangerous!