Initial population

git-svn-id: https://svn.o-hand.com/repos/poky@2 311d38ba-8fff-0310-9ca6-ca027cbcb966

bitbake/bin/bbimage

@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2003  Chris Larson
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+# 
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA 02111-1307 USA. 
+
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+import bb
+from bb import *
+
+__version__ = 1.0
+type = "jffs2"
+cfg_bb = data.init()
+cfg_oespawn = data.init()
+
+
+def usage():
+    print "Usage: bbimage [options ...]"
+    print "Creates an image for a target device from a root filesystem,"
+    print "obeying configuration parameters from the BitBake"
+    print "configuration files, thereby easing handling of deviceisms."
+    print ""
+    print "  %s\t\t%s" % ("-r [arg], --root [arg]", "root directory (default=${IMAGE_ROOTFS})")
+    print "  %s\t\t%s" % ("-t [arg], --type [arg]", "image type (jffs2[default], cramfs)")
+    print "  %s\t\t%s" % ("-n [arg], --name [arg]", "image name (override IMAGE_NAME variable)")
+    print "  %s\t\t%s" % ("-v, --version", "output version information and exit")
+    sys.exit(0)
+
+def version():
+    print "BitBake Build Tool Core version %s" % bb.__version__
+    print "BBImage version %s" % __version__
+
+def emit_bb(d, base_d = {}):
+    for v in d.keys():
+        if d[v] != base_d[v]:
+            data.emit_var(v, d)
+
+def getopthash(l):
+    h = {}
+    for (opt, val) in l:
+        h[opt] = val
+    return h
+
+import getopt
+try:
+    (opts, args) = getopt.getopt(sys.argv[1:], 'vr:t:e:n:', [ 'version', 'root=', 'type=', 'bbfile=', 'name=' ])
+except getopt.GetoptError:
+    usage()
+
+# handle opts
+opthash = getopthash(opts)
+
+if '--version' in opthash or '-v' in opthash:
+    version()
+    sys.exit(0)
+
+try:
+    cfg_bb = parse.handle(os.path.join('conf', 'bitbake.conf'), cfg_bb)
+except IOError:
+    fatal("Unable to open bitbake.conf")
+
+# sanity check
+if cfg_bb is None:
+    fatal("Unable to open/parse %s" % os.path.join('conf', 'bitbake.conf'))
+    usage(1)
+
+rootfs = None
+extra_files = []
+
+if '--root' in opthash:
+    rootfs = opthash['--root']
+if '-r' in opthash:
+    rootfs = opthash['-r']
+
+if '--type' in opthash:
+    type = opthash['--type']
+if '-t' in opthash:
+    type = opthash['-t']
+
+if '--bbfile' in opthash:
+    extra_files.append(opthash['--bbfile'])
+if '-e' in opthash:
+    extra_files.append(opthash['-e'])
+
+for f in extra_files:
+    try:
+        cfg_bb = parse.handle(f, cfg_bb)
+    except IOError:
+        print "unable to open %s" % f
+
+if not rootfs:
+    rootfs = data.getVar('IMAGE_ROOTFS', cfg_bb, 1)
+
+if not rootfs:
+    bb.fatal("IMAGE_ROOTFS not defined")
+
+data.setVar('IMAGE_ROOTFS', rootfs, cfg_bb)
+
+from copy import copy, deepcopy
+localdata = data.createCopy(cfg_bb)
+
+overrides = data.getVar('OVERRIDES', localdata)
+if not overrides:
+    bb.fatal("OVERRIDES not defined.")
+data.setVar('OVERRIDES', '%s:%s' % (overrides, type), localdata)
+data.update_data(localdata)
+data.setVar('OVERRIDES', overrides, localdata)
+
+if '-n' in opthash:
+    data.setVar('IMAGE_NAME', opthash['-n'], localdata)
+if '--name' in opthash:
+    data.setVar('IMAGE_NAME', opthash['--name'], localdata)
+
+topdir = data.getVar('TOPDIR', localdata, 1) or os.getcwd()
+
+cmd = data.getVar('IMAGE_CMD', localdata, 1)
+if not cmd:
+    bb.fatal("IMAGE_CMD not defined")
+
+outdir = data.getVar('DEPLOY_DIR_IMAGE', localdata, 1)
+if not outdir:
+    bb.fatal('DEPLOY_DIR_IMAGE not defined')
+mkdirhier(outdir)
+
+#depends = data.getVar('IMAGE_DEPENDS', localdata, 1) or ""
+#if depends:
+#       bb.note("Spawning bbmake to satisfy dependencies: %s" % depends)
+#       ret = os.system('bbmake %s' % depends)
+#       if ret != 0:
+#           bb.error("executing bbmake to satisfy dependencies")
+
+bb.note("Executing %s" % cmd)
+data.setVar('image_cmd', cmd, localdata)
+data.setVarFlag('image_cmd', 'func', 1, localdata)
+try:
+    bb.build.exec_func('image_cmd', localdata)
+except bb.build.FuncFailed:
+    sys.exit(1)
+#ret = os.system(cmd)
+#sys.exit(ret)

bitbake/bin/bitbake

@@ -0,0 +1,927 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2003, 2004  Chris Larson
+# Copyright (C) 2003, 2004  Phil Blundell
+# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
+# Copyright (C) 2005        Holger Hans Peter Freyther
+# Copyright (C) 2005        ROAD GmbH
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA 02111-1307 USA.
+
+import sys, os, getopt, glob, copy, os.path, re, time
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+import bb
+from bb import utils, data, parse, debug, event, fatal
+from sets import Set
+import itertools, optparse
+
+parsespin = itertools.cycle( r'|/-\\' )
+bbdebug = 0
+
+__version__ = "1.3.2"
+
+#============================================================================#
+# BBParsingStatus
+#============================================================================#
+class BBParsingStatus:
+    """
+    The initial idea for this status class is to use the data when it is
+    already loaded instead of loading it from various place over and over
+    again.
+    """
+
+    def __init__(self):
+        self.cache_dirty = False
+        self.providers   = {}
+        self.bbfile_priority = {}
+        self.bbfile_config_priorities = []
+        self.ignored_depedencies = None
+        self.possible_world = []
+        self.world_target = Set()
+        self.pkg_pn = {}
+        self.pkg_fn = {}
+        self.pkg_pvpr = {}
+        self.pkg_dp = {}
+        self.pn_provides = {}
+        self.all_depends = Set()
+
+    def handle_bb_data(self, file_name, bb_data, cached):
+        """
+        We will fill the dictionaries with the stuff we
+        need for building the tree more fast
+        """
+        if bb_data == None:
+            return
+
+        if not cached:
+            self.cache_dirty = True
+
+        pn       = bb.data.getVar('PN', bb_data, True)
+        pv       = bb.data.getVar('PV', bb_data, True)
+        pr       = bb.data.getVar('PR', bb_data, True)
+        dp       = int(bb.data.getVar('DEFAULT_PREFERENCE', bb_data, True) or "0")
+        provides = Set([pn] + (bb.data.getVar("PROVIDES", bb_data, 1) or "").split())
+        depends  = (bb.data.getVar("DEPENDS", bb_data, True) or "").split()
+
+
+        # build PackageName to FileName lookup table
+        if pn not in self.pkg_pn:
+            self.pkg_pn[pn] = []
+        self.pkg_pn[pn].append(file_name)
+
+        # build FileName to PackageName lookup table
+        self.pkg_fn[file_name] = pn
+        self.pkg_pvpr[file_name] = (pv,pr)
+        self.pkg_dp[file_name] = dp
+
+        # Build forward and reverse provider hashes
+        # Forward: virtual -> [filenames]
+        # Reverse: PN -> [virtuals]
+        if pn not in self.pn_provides:
+            self.pn_provides[pn] = Set()
+        self.pn_provides[pn] |= provides
+
+        for provide in provides:
+            if provide not in self.providers:
+                self.providers[provide] = []
+            self.providers[provide].append(file_name)
+
+        for dep in depends:
+            self.all_depends.add(dep)
+
+        # Collect files we may need for possible world-dep
+        # calculations
+        if not bb.data.getVar('BROKEN', bb_data, True) and not bb.data.getVar('EXCLUDE_FROM_WORLD', bb_data, True):
+            self.possible_world.append(file_name)
+
+
+#============================================================================#
+# BBStatistics
+#============================================================================#
+class BBStatistics:
+    """
+    Manage build statistics for one run
+    """
+    def __init__(self ):
+        self.attempt = 0
+        self.success = 0
+        self.fail = 0
+        self.deps = 0
+
+    def show( self ):
+        print "Build statistics:"
+        print "  Attempted builds: %d" % self.attempt
+        if self.fail:
+            print "  Failed builds: %d" % self.fail
+        if self.deps:
+            print "  Dependencies not satisfied: %d" % self.deps
+        if self.fail or self.deps: return 1
+        else: return 0
+
+
+#============================================================================#
+# BBOptions
+#============================================================================#
+class BBConfiguration( object ):
+    """
+    Manages build options and configurations for one run
+    """
+    def __init__( self, options ):
+        for key, val in options.__dict__.items():
+            setattr( self, key, val )
+        self.data = data.init()
+
+#============================================================================#
+# BBCooker
+#============================================================================#
+class BBCooker:
+    """
+    Manages one bitbake build run
+    """
+
+    ParsingStatus = BBParsingStatus     # make it visible from the shell
+    Statistics = BBStatistics           # make it visible from the shell
+
+    def __init__( self ):
+        self.build_cache_fail = []
+        self.build_cache = []
+        self.building_list = []
+        self.build_path = []
+        self.consider_msgs_cache = []
+        self.preferred = {}
+        self.stats = BBStatistics()
+        self.status = None
+
+        self.pkgdata = None
+        self.cache = None
+
+    def tryBuildPackage( self, fn, item, the_data ):
+        """Build one package"""
+        bb.event.fire(bb.event.PkgStarted(item, the_data))
+        try:
+            self.stats.attempt += 1
+            if self.configuration.force:
+                bb.data.setVarFlag('do_%s' % self.configuration.cmd, 'force', 1, the_data)
+            if not self.configuration.dry_run:
+                bb.build.exec_task('do_%s' % self.configuration.cmd, the_data)
+            bb.event.fire(bb.event.PkgSucceeded(item, the_data))
+            self.build_cache.append(fn)
+            return True
+        except bb.build.FuncFailed:
+            self.stats.fail += 1
+            bb.error("task stack execution failed")
+            bb.event.fire(bb.event.PkgFailed(item, the_data))
+            self.build_cache_fail.append(fn)
+            raise
+        except bb.build.EventException, e:
+            self.stats.fail += 1
+            event = e.args[1]
+            bb.error("%s event exception, aborting" % bb.event.getName(event))
+            bb.event.fire(bb.event.PkgFailed(item, the_data))
+            self.build_cache_fail.append(fn)
+            raise
+
+    def tryBuild( self, fn, virtual ):
+        """Build a provider and its dependencies"""
+        if fn in self.building_list:
+            bb.error("%s depends on itself (eventually)" % fn)
+            bb.error("upwards chain is: %s" % (" -> ".join(self.build_path)))
+            return False
+
+        the_data = self.pkgdata[fn]
+        item = self.status.pkg_fn[fn]
+
+        self.building_list.append(fn)
+
+        pathstr = "%s (%s)" % (item, virtual)
+        self.build_path.append(pathstr)
+
+        depends_list = (bb.data.getVar('DEPENDS', the_data, 1) or "").split()
+        if self.configuration.verbose:
+            bb.note("current path: %s" % (" -> ".join(self.build_path)))
+            bb.note("dependencies for %s are: %s" % (item, " ".join(depends_list)))
+
+        try:
+            failed = False
+
+            depcmd = self.configuration.cmd
+            bbdepcmd = bb.data.getVarFlag('do_%s' % self.configuration.cmd, 'bbdepcmd', the_data)
+            if bbdepcmd is not None:
+                if bbdepcmd == "":
+                    depcmd = None
+                else:
+                    depcmd = bbdepcmd
+
+            if depcmd:
+                oldcmd = self.configuration.cmd
+                self.configuration.cmd = depcmd
+
+            for dependency in depends_list:
+                if dependency in self.status.ignored_dependencies:
+                    continue
+                if not depcmd:
+                    continue
+                if self.buildProvider( dependency ) == 0:
+                    bb.error("dependency %s (for %s) not satisfied" % (dependency,item))
+                    failed = True
+                    if self.configuration.abort:
+                        break
+
+            if depcmd:
+                self.configuration.cmd = oldcmd
+
+            if failed:
+                self.stats.deps += 1
+                return False
+
+            if bb.build.stamp_is_current('do_%s' % self.configuration.cmd, the_data):
+                self.build_cache.append(fn)
+                return True
+
+            return self.tryBuildPackage( fn, item, the_data )
+
+        finally:
+            self.building_list.remove(fn)
+            self.build_path.remove(pathstr)
+
+    def findBestProvider( self, pn, pkg_pn = None):
+        """
+        If there is a PREFERRED_VERSION, find the highest-priority bbfile
+        providing that version.  If not, find the latest version provided by
+        an bbfile in the highest-priority set.
+        """
+        if not pkg_pn:
+            pkg_pn = self.status.pkg_pn
+
+        files = pkg_pn[pn]
+        priorities = {}
+        for f in files:
+            priority = self.status.bbfile_priority[f]
+            if priority not in priorities:
+                priorities[priority] = []
+            priorities[priority].append(f)
+        p_list = priorities.keys()
+        p_list.sort(lambda a, b: a - b)
+        tmp_pn = []
+        for p in p_list:
+            tmp_pn = [priorities[p]] + tmp_pn
+
+        preferred_file = None
+
+        preferred_v = bb.data.getVar('PREFERRED_VERSION_%s' % pn, self.configuration.data, 1)
+        if preferred_v:
+            m = re.match('(.*)_(.*)', preferred_v)
+            if m:
+                preferred_v = m.group(1)
+                preferred_r = m.group(2)
+            else:
+                preferred_r = None
+
+            for file_set in tmp_pn:
+                for f in file_set:
+                    pv,pr = self.status.pkg_pvpr[f]
+                    if preferred_v == pv and (preferred_r == pr or preferred_r == None):
+                        preferred_file = f
+                        preferred_ver = (pv, pr)
+                        break
+                if preferred_file:
+                    break;
+            if preferred_r:
+                pv_str = '%s-%s' % (preferred_v, preferred_r)
+            else:
+                pv_str = preferred_v
+            if preferred_file is None:
+                bb.note("preferred version %s of %s not available" % (pv_str, pn))
+            else:
+                bb.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s" % (preferred_file, pv_str, pn))
+
+        # get highest priority file set
+        files = tmp_pn[0]
+        latest = None
+        latest_p = 0
+        latest_f = None
+        for file_name in files:
+            pv,pr = self.status.pkg_pvpr[file_name]
+            dp = self.status.pkg_dp[file_name]
+
+            if (latest is None) or ((latest_p == dp) and (utils.vercmp(latest, (pv, pr)) < 0)) or (dp > latest_p):
+                latest = (pv, pr)
+                latest_f = file_name
+                latest_p = dp
+        if preferred_file is None:
+            preferred_file = latest_f
+            preferred_ver = latest
+
+        return (latest,latest_f,preferred_ver, preferred_file)
+
+    def showVersions( self ):
+        pkg_pn = self.status.pkg_pn
+        preferred_versions = {}
+        latest_versions = {}
+
+        # Sort by priority
+        for pn in pkg_pn.keys():
+            (last_ver,last_file,pref_ver,pref_file) = self.findBestProvider(pn)
+            preferred_versions[pn] = (pref_ver, pref_file)
+            latest_versions[pn] = (last_ver, last_file)
+
+        pkg_list = pkg_pn.keys()
+        pkg_list.sort()
+
+        for p in pkg_list:
+            pref = preferred_versions[p]
+            latest = latest_versions[p]
+
+            if pref != latest:
+                prefstr = pref[0][0] + "-" + pref[0][1]
+            else:
+                prefstr = ""
+
+            print "%-30s %20s %20s" % (p, latest[0][0] + "-" + latest[0][1],
+                                        prefstr)
+
+    def showEnvironment( self ):
+        """Show the outer or per-package environment"""
+        if self.configuration.buildfile:
+            try:
+                self.configuration.data, fromCache = self.load_bbfile( self.configuration.buildfile )
+            except IOError, e:
+                fatal("Unable to read %s: %s" % ( self.configuration.buildfile, e ))
+            except Exception, e:
+                fatal("%s" % e)
+        # emit variables and shell functions
+        try:
+            data.update_data( self.configuration.data )
+            data.emit_env(sys.__stdout__, self.configuration.data, True)
+        except Exception, e:
+            fatal("%s" % e)
+        # emit the metadata which isnt valid shell
+        for e in self.configuration.data.keys():
+            if data.getVarFlag( e, 'python', self.configuration.data ):
+                sys.__stdout__.write("\npython %s () {\n%s}\n" % (e, data.getVar(e, self.configuration.data, 1)))
+
+    def buildProvider( self, item ):
+        fn = None
+
+        discriminated = False
+
+        if item not in self.status.providers:
+            bb.error("Nothing provides %s" % item)
+            return 0
+
+        all_p = self.status.providers[item]
+
+        for p in all_p:
+            if p in self.build_cache:
+                bb.debug(1, "already built %s in this run\n" % p)
+                return 1
+
+        eligible = []
+        preferred_versions = {}
+
+        # Collate providers by PN
+        pkg_pn = {}
+        for p in all_p:
+            pn = self.status.pkg_fn[p]
+            if pn not in pkg_pn:
+                pkg_pn[pn] = []
+            pkg_pn[pn].append(p)
+
+        bb.debug(1, "providers for %s are: %s" % (item, pkg_pn.keys()))
+
+        for pn in pkg_pn.keys():
+            preferred_versions[pn] = self.findBestProvider(pn, pkg_pn)[2:4]
+            eligible.append(preferred_versions[pn][1])
+
+        for p in eligible:
+            if p in self.build_cache_fail:
+                bb.debug(1, "rejecting already-failed %s" % p)
+                eligible.remove(p)
+
+        if len(eligible) == 0:
+            bb.error("no eligible providers for %s" % item)
+            return 0
+
+        # look to see if one of them is already staged, or marked as preferred.
+        # if so, bump it to the head of the queue
+        for p in all_p:
+            the_data = self.pkgdata[p]
+            pn = bb.data.getVar('PN', the_data, 1)
+            pv = bb.data.getVar('PV', the_data, 1)
+            pr = bb.data.getVar('PR', the_data, 1)
+            tmpdir = bb.data.getVar('TMPDIR', the_data, 1)
+            stamp = '%s/stamps/%s-%s-%s.do_populate_staging' % (tmpdir, pn, pv, pr)
+            if os.path.exists(stamp):
+                (newvers, fn) = preferred_versions[pn]
+                if not fn in eligible:
+                    # package was made ineligible by already-failed check
+                    continue
+                oldver = "%s-%s" % (pv, pr)
+                newver = '-'.join(newvers)
+                if (newver != oldver):
+                    extra_chat = "; upgrading from %s to %s" % (oldver, newver)
+                else:
+                    extra_chat = ""
+                if self.configuration.verbose:
+                    bb.note("selecting already-staged %s to satisfy %s%s" % (pn, item, extra_chat))
+                eligible.remove(fn)
+                eligible = [fn] + eligible
+                discriminated = True
+                break
+
+        prefervar = bb.data.getVar('PREFERRED_PROVIDER_%s' % item, self.configuration.data, 1)
+        if prefervar:
+            self.preferred[item] = prefervar
+
+        if item in self.preferred:
+            for p in eligible:
+                pn = self.status.pkg_fn[p]
+                if self.preferred[item] == pn:
+                    if self.configuration.verbose:
+                        bb.note("selecting %s to satisfy %s due to PREFERRED_PROVIDERS" % (pn, item))
+                    eligible.remove(p)
+                    eligible = [p] + eligible
+                    discriminated = True
+                    break
+
+        if len(eligible) > 1 and discriminated == False:
+            if item not in self.consider_msgs_cache:
+                providers_list = []
+                for fn in eligible:
+                    providers_list.append(self.status.pkg_fn[fn])
+                bb.note("multiple providers are available (%s);" % ", ".join(providers_list))
+                bb.note("consider defining PREFERRED_PROVIDER_%s" % item)
+            self.consider_msgs_cache.append(item)
+
+
+        # run through the list until we find one that we can build
+        for fn in eligible:
+            bb.debug(2, "selecting %s to satisfy %s" % (fn, item))
+            if self.tryBuild(fn, item):
+                return 1
+
+        bb.note("no buildable providers for %s" % item)
+        return 0
+
+    def buildDepgraph( self ):
+        all_depends = self.status.all_depends
+        pn_provides = self.status.pn_provides
+
+        def calc_bbfile_priority(filename):
+            for (regex, pri) in self.status.bbfile_config_priorities:
+                if regex.match(filename):
+                    return pri
+            return 0
+
+        # Handle PREFERRED_PROVIDERS
+        for p in (bb.data.getVar('PREFERRED_PROVIDERS', self.configuration.data, 1) or "").split():
+            (providee, provider) = p.split(':')
+            if providee in self.preferred and self.preferred[providee] != provider:
+                bb.error("conflicting preferences for %s: both %s and %s specified" % (providee, provider, self.preferred[providee]))
+            self.preferred[providee] = provider
+
+        # Calculate priorities for each file
+        for p in self.pkgdata.keys():
+            self.status.bbfile_priority[p] = calc_bbfile_priority(p)
+
+        # Build package list for "bitbake world"
+        bb.debug(1, "collating packages for \"world\"")
+        for f in self.status.possible_world:
+            terminal = True
+            pn = self.status.pkg_fn[f]
+
+            for p in pn_provides[pn]:
+                if p.startswith('virtual/'):
+                    bb.debug(2, "skipping %s due to %s provider starting with virtual/" % (f, p))
+                    terminal = False
+                    break
+                for pf in self.status.providers[p]:
+                    if self.status.pkg_fn[pf] != pn:
+                        bb.debug(2, "skipping %s due to both us and %s providing %s" % (f, pf, p))
+                        terminal = False
+                        break
+            if terminal:
+                self.status.world_target.add(pn)
+
+            # drop reference count now
+            self.status.possible_world = None
+            self.status.all_depends    = None
+
+    def myProgressCallback( self, x, y, f, file_data, from_cache ):
+        # feed the status with new input
+        self.status.handle_bb_data(f, file_data, from_cache)
+
+        if bbdebug > 0:
+            return
+        if os.isatty(sys.stdout.fileno()):
+            sys.stdout.write("\rNOTE: Handling BitBake files: %s (%04d/%04d) [%2d %%]" % ( parsespin.next(), x, y, x*100/y ) )
+            sys.stdout.flush()
+        else:
+            if x == 1:
+                sys.stdout.write("Parsing .bb files, please wait...")
+                sys.stdout.flush()
+            if x == y:
+                sys.stdout.write("done.")
+                sys.stdout.flush()
+
+    def interactiveMode( self ):
+        """Drop off into a shell"""
+        try:
+            from bb import shell
+        except ImportError, details:
+            bb.fatal("Sorry, shell not available (%s)" % details )
+        else:
+            bb.data.update_data( self.configuration.data )
+            shell.start( self )
+            sys.exit( 0 )
+
+    def parseConfigurationFile( self, afile ):
+        try:
+            self.configuration.data = bb.parse.handle( afile, self.configuration.data )
+        except IOError:
+            bb.fatal( "Unable to open %s" % afile )
+        except bb.parse.ParseError, details:
+            bb.fatal( "Unable to parse %s (%s)" % (afile, details) )
+
+    def handleCollections( self, collections ):
+        """Handle collections"""
+        if collections:
+            collection_list = collections.split()
+            for c in collection_list:
+                regex = bb.data.getVar("BBFILE_PATTERN_%s" % c, self.configuration.data, 1)
+                if regex == None:
+                    bb.error("BBFILE_PATTERN_%s not defined" % c)
+                    continue
+                priority = bb.data.getVar("BBFILE_PRIORITY_%s" % c, self.configuration.data, 1)
+                if priority == None:
+                    bb.error("BBFILE_PRIORITY_%s not defined" % c)
+                    continue
+                try:
+                    cre = re.compile(regex)
+                except re.error:
+                    bb.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression" % (c, regex))
+                    continue
+                try:
+                    pri = int(priority)
+                    self.status.bbfile_config_priorities.append((cre, pri))
+                except ValueError:
+                    bb.error("invalid value for BBFILE_PRIORITY_%s: \"%s\"" % (c, priority))
+
+
+    def cook( self, configuration, args ):
+        self.configuration = configuration
+
+        if not self.configuration.cmd:
+            self.configuration.cmd = "build"
+
+        if self.configuration.debug:
+            bb.debug_level = self.configuration.debug
+
+        self.configuration.data = bb.data.init()
+
+        for f in self.configuration.file:
+            self.parseConfigurationFile( f )
+
+        self.parseConfigurationFile( os.path.join( "conf", "bitbake.conf" ) )
+
+        if self.configuration.show_environment:
+            self.showEnvironment()
+            sys.exit( 0 )
+
+        # inject custom variables
+        if not bb.data.getVar("BUILDNAME", self.configuration.data):
+            bb.data.setVar("BUILDNAME", os.popen('date +%Y%m%d%H%M').readline().strip(), self.configuration.data)
+        bb.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S',time.gmtime()),self.configuration.data)
+
+        buildname = bb.data.getVar("BUILDNAME", self.configuration.data)
+
+        if self.configuration.interactive:
+            self.interactiveMode()
+
+        if self.configuration.buildfile is not None:
+            bf = os.path.abspath( self.configuration.buildfile )
+            try:
+                bbfile_data = bb.parse.handle(bf, self.configuration.data)
+            except IOError:
+                bb.fatal("Unable to open %s" % bf)
+
+            item = bb.data.getVar('PN', bbfile_data, 1)
+            try:
+                self.tryBuildPackage( bf, item, bbfile_data )
+            except bb.build.EventException:
+                bb.error( "Build of '%s' failed" % item )
+
+            sys.exit( self.stats.show() )
+
+        # initialise the parsing status now we know we will need deps
+        self.status = BBParsingStatus()
+
+        ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
+        self.status.ignored_dependencies = Set( ignore.split() )
+
+        self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
+
+        pkgs_to_build = None
+        if args:
+            if not pkgs_to_build:
+                pkgs_to_build = []
+            pkgs_to_build.extend(args)
+        if not pkgs_to_build:
+                bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, 1)
+                if bbpkgs:
+                        pkgs_to_build = bbpkgs.split()
+        if not pkgs_to_build and not self.configuration.show_versions \
+                             and not self.configuration.interactive \
+                             and not self.configuration.show_environment:
+                print "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help'"
+                print "for usage information."
+                sys.exit(0)
+
+        # Import Psyco if available and not disabled
+        if not self.configuration.disable_psyco:
+            try:
+                import psyco
+            except ImportError:
+                if bbdebug == 0:
+                    bb.note("Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
+            else:
+                psyco.bind( self.collect_bbfiles )
+        else:
+            bb.note("You have disabled Psyco. This decreases performance.")
+
+        try:
+            bb.debug(1, "collecting .bb files")
+            self.collect_bbfiles( self.myProgressCallback )
+            bb.debug(1, "parsing complete")
+            if bbdebug == 0:
+                print
+            if self.configuration.parse_only:
+                print "Requested parsing .bb files only.  Exiting."
+                return
+
+            bb.data.update_data( self.configuration.data )
+            self.buildDepgraph()
+
+            if self.configuration.show_versions:
+                self.showVersions()
+                sys.exit( 0 )
+            if 'world' in pkgs_to_build:
+                pkgs_to_build.remove('world')
+                for t in self.status.world_target:
+                    pkgs_to_build.append(t)
+
+            bb.event.fire(bb.event.BuildStarted(buildname, pkgs_to_build, self.configuration.data))
+
+            for k in pkgs_to_build:
+                failed = False
+                try:
+                    if self.buildProvider( k ) == 0:
+                        # already diagnosed
+                        failed = True
+                except bb.build.EventException:
+                    bb.error("Build of " + k + " failed")
+                    failed = True
+
+                if failed:
+                    if self.configuration.abort:
+                        sys.exit(1)
+
+            bb.event.fire(bb.event.BuildCompleted(buildname, pkgs_to_build, self.configuration.data))
+
+            sys.exit( self.stats.show() )
+
+        except KeyboardInterrupt:
+            print "\nNOTE: KeyboardInterrupt - Build not completed."
+            sys.exit(1)
+
+    def get_bbfiles( self, path = os.getcwd() ):
+        """Get list of default .bb files by reading out the current directory"""
+        contents = os.listdir(path)
+        bbfiles = []
+        for f in contents:
+            (root, ext) = os.path.splitext(f)
+            if ext == ".bb":
+                bbfiles.append(os.path.abspath(os.path.join(os.getcwd(),f)))
+        return bbfiles
+
+    def find_bbfiles( self, path ):
+        """Find all the .bb files in a directory (uses find)"""
+        findcmd = 'find ' + path + ' -name *.bb | grep -v SCCS/'
+        try:
+            finddata = os.popen(findcmd)
+        except OSError:
+            return []
+        return finddata.readlines()
+
+    def deps_clean(self, d):
+        depstr = data.getVar('__depends', d)
+        if depstr:
+            deps = depstr.split(" ")
+            for dep in deps:
+                (f,old_mtime_s) = dep.split("@")
+                old_mtime = int(old_mtime_s)
+                new_mtime = parse.cached_mtime(f)
+                if (new_mtime > old_mtime):
+                    return False
+        return True
+
+    def load_bbfile( self, bbfile ):
+        """Load and parse one .bb build file"""
+
+        if not self.cache in [None, '']:
+            # get the times
+            cache_mtime = data.init_db_mtime(self.cache, bbfile)
+            file_mtime = parse.cached_mtime(bbfile)
+
+            if file_mtime > cache_mtime:
+                #print " : '%s' dirty. reparsing..." % bbfile
+                pass
+            else:
+                #print " : '%s' clean. loading from cache..." % bbfile
+                cache_data = data.init_db( self.cache, bbfile, False )
+                if self.deps_clean(cache_data):
+                    return cache_data, True
+
+        topdir = data.getVar('TOPDIR', self.configuration.data)
+        if not topdir:
+            topdir = os.path.abspath(os.getcwd())
+            # set topdir to here
+            data.setVar('TOPDIR', topdir, self.configuration)
+        bbfile = os.path.abspath(bbfile)
+        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
+        # expand tmpdir to include this topdir
+        data.setVar('TMPDIR', data.getVar('TMPDIR', self.configuration.data, 1) or "", self.configuration.data)
+        # set topdir to location of .bb file
+        topdir = bbfile_loc
+        #data.setVar('TOPDIR', topdir, cfg)
+        # go there
+        oldpath = os.path.abspath(os.getcwd())
+        os.chdir(topdir)
+        bb = data.init_db(self.cache,bbfile, True, self.configuration.data)
+        try:
+            parse.handle(bbfile, bb) # read .bb data
+            if not self.cache in [None, '']:
+                bb.commit(parse.cached_mtime(bbfile)) # write cache
+            os.chdir(oldpath)
+            return bb, False
+        finally:
+            os.chdir(oldpath)
+
+    def collect_bbfiles( self, progressCallback ):
+        """Collect all available .bb build files"""
+        self.cb = progressCallback
+        parsed, cached, skipped, masked = 0, 0, 0, 0
+        self.cache   = bb.data.getVar( "CACHE", self.configuration.data, 1 )
+        self.pkgdata = data.pkgdata( not self.cache in [None, ''], self.cache, self.configuration.data )
+
+        if not self.cache in [None, '']:
+            if self.cb is not None:
+                print "NOTE: Using cache in '%s'" % self.cache
+            try:
+                os.stat( self.cache )
+            except OSError:
+                bb.mkdirhier( self.cache )
+        else:
+            if self.cb is not None:
+                print "NOTE: Not using a cache. Set CACHE = <directory> to enable."
+        files = (data.getVar( "BBFILES", self.configuration.data, 1 ) or "").split()
+        data.setVar("BBFILES", " ".join(files), self.configuration.data)
+
+        if not len(files):
+            files = self.get_bbfiles()
+
+        if not len(files):
+            bb.error("no files to build.")
+
+        newfiles = []
+        for f in files:
+            if os.path.isdir(f):
+                dirfiles = self.find_bbfiles(f)
+                if dirfiles:
+                    newfiles += dirfiles
+                    continue
+            newfiles += glob.glob(f) or [ f ]
+
+        bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1) or ""
+        try:
+            bbmask_compiled = re.compile(bbmask)
+        except sre_constants.error:
+            bb.fatal("BBMASK is not a valid regular expression.")
+
+        for i in xrange( len( newfiles ) ):
+            f = newfiles[i]
+            if bbmask and bbmask_compiled.search(f):
+                bb.debug(1, "bbmake: skipping %s" % f)
+                masked += 1
+                continue
+            debug(1, "bbmake: parsing %s" % f)
+
+            # read a file's metadata
+            try:
+                bb_data, fromCache = self.load_bbfile(f)
+                if fromCache: cached += 1
+                else: parsed += 1
+                deps = None
+                if bb_data is not None:
+                    # allow metadata files to add items to BBFILES
+                    #data.update_data(self.pkgdata[f])
+                    addbbfiles = data.getVar('BBFILES', bb_data) or None
+                    if addbbfiles:
+                        for aof in addbbfiles.split():
+                            if not files.count(aof):
+                                if not os.path.isabs(aof):
+                                    aof = os.path.join(os.path.dirname(f),aof)
+                                files.append(aof)
+                    for var in bb_data.keys():
+                        if data.getVarFlag(var, "handler", bb_data) and data.getVar(var, bb_data):
+                            event.register(data.getVar(var, bb_data))
+                    self.pkgdata[f] = bb_data
+
+                # now inform the caller
+                if self.cb is not None:
+                    self.cb( i + 1, len( newfiles ), f, bb_data, fromCache )
+
+            except IOError, e:
+                bb.error("opening %s: %s" % (f, e))
+                pass
+            except bb.parse.SkipPackage:
+                skipped += 1
+                pass
+            except KeyboardInterrupt:
+                raise
+            except Exception, e:
+                bb.error("%s while parsing %s" % (e, f))
+
+        if self.cb is not None:
+            print "\rNOTE: Parsing finished. %d cached, %d parsed, %d skipped, %d masked." % ( cached, parsed, skipped, masked ),
+
+#============================================================================#
+# main
+#============================================================================#
+
+if __name__ == "__main__":
+
+    parser = optparse.OptionParser( version = "BitBake Build Tool Core version %s, %%prog version %s" % ( bb.__version__, __version__ ),
+    usage = """%prog [options] [package ...]
+
+Executes the specified task (default is 'build') for a given set of BitBake files.
+It expects that BBFILES is defined, which is a space seperated list of files to
+be executed.  BBFILES does support wildcards.
+Default BBFILES are the .bb files in the current directory.""" )
+
+    parser.add_option( "-b", "--buildfile", help = "execute the task against this .bb file, rather than a package from BBFILES.",
+               action = "store", dest = "buildfile", default = None )
+
+    parser.add_option( "-k", "--continue", help = "continue as much as possible after an error. While the target that failed, and those that depend on it, cannot be remade, the other dependencies of these targets can be processed all the same.",
+               action = "store_false", dest = "abort", default = True )
+
+    parser.add_option( "-f", "--force", help = "force run of specified cmd, regardless of stamp status",
+               action = "store_true", dest = "force", default = False )
+
+    parser.add_option( "-i", "--interactive", help = "drop into the interactive mode.",
+               action = "store_true", dest = "interactive", default = False )
+
+    parser.add_option( "-c", "--cmd", help = "Specify task to execute. Note that this only executes the specified task for the providee and the packages it depends on, i.e. 'compile' does not implicitly call stage for the dependencies (IOW: use only if you know what you are doing)",
+               action = "store", dest = "cmd", default = "build" )
+
+    parser.add_option( "-r", "--read", help = "read the specified file before bitbake.conf",
+               action = "append", dest = "file", default = [] )
+
+    parser.add_option( "-v", "--verbose", help = "output more chit-chat to the terminal",
+               action = "store_true", dest = "verbose", default = False )
+
+    parser.add_option( "-D", "--debug", help = "Increase the debug level",
+               action = "count", dest="debug", default = 0)
+
+    parser.add_option( "-n", "--dry-run", help = "don't execute, just go through the motions",
+               action = "store_true", dest = "dry_run", default = False )
+
+    parser.add_option( "-p", "--parse-only", help = "quit after parsing the BB files (developers only)",
+               action = "store_true", dest = "parse_only", default = False )
+
+    parser.add_option( "-d", "--disable-psyco", help = "disable using the psyco just-in-time compiler (not recommended)",
+               action = "store_true", dest = "disable_psyco", default = False )
+
+    parser.add_option( "-s", "--show-versions", help = "show current and preferred versions of all packages",
+               action = "store_true", dest = "show_versions", default = False )
+
+    parser.add_option( "-e", "--environment", help = "show the global or per-package environment (this is what used to be bbread)",
+               action = "store_true", dest = "show_environment", default = False )
+
+    options, args = parser.parse_args( sys.argv )
+
+    cooker = BBCooker()
+    cooker.cook( BBConfiguration( options ), args[1:] )

bitbake/bin/bitdoc

@@ -0,0 +1,529 @@
+#!/usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (C) 2005 Holger Hans Peter Freyther
+#
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+# THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+#
+
+import optparse, os, sys
+
+# bitbake
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+import bb
+from   bb import make
+from   string import split, join
+
+__version__ = "0.0.2"
+
+class HTMLFormatter:
+    """
+    Simple class to help to generate some sort of HTML files. It is
+    quite inferior solution compared to docbook, gtkdoc, doxygen but it
+    should work for now.
+    We've a global introduction site (index.html) and then one site for
+    the list of keys (alphabetical sorted) and one for the list of groups,
+    one site for each key with links to the relations and groups.
+
+        index.html
+        keys.html
+        groups.html
+        groupNAME.html
+        keyNAME.html
+    """
+
+    def replace(self, text, *pairs):
+        """
+        From pydoc... almost identical at least
+        """
+        while pairs:
+            (a,b) = pairs[0]
+            text = join(split(text, a), b)
+            pairs = pairs[1:]
+        return text
+    def escape(self, text):
+        """
+        Escape string to be conform HTML
+        """
+        return self.replace(text, 
+                            ('&', '&'), 
+                            ('<', '&lt;' ),
+                            ('>', '&gt;' ) )
+    def createNavigator(self):
+        """
+        Create the navgiator
+        """
+        return """<table class="navigation" width="100%" summary="Navigation header" cellpadding="2" cellspacing="2">
+<tr valign="middle">
+<td><a accesskey="g" href="index.html">Home</a></td>
+<td><a accesskey="n" href="groups.html">Groups</a></td>
+<td><a accesskey="u" href="keys.html">Keys</a></td>
+</tr></table>
+"""
+
+    def relatedKeys(self, item):
+        """
+        Create HTML to link to foreign keys
+        """
+
+        if len(item.related()) == 0:
+            return ""
+
+        txt = "<p><b>See also:</b><br>"
+        for it in item.related():
+            txt += """<a href="key%s.html">%s</a>, """ % (it, it)
+
+        return txt
+
+    def groups(self,item):
+        """
+        Create HTML to link to related groups
+        """
+
+        if len(item.groups()) == 0:
+            return ""
+
+
+        txt = "<p><b>Seel also:</b><br>"
+        for group in item.groups():
+            txt += """<a href="group%s.html">%s</a>, """ % (group,group)
+
+        return txt
+
+
+    def createKeySite(self,item):
+        """
+        Create a site for a key. It contains the header/navigator, a heading,
+        the description, links to related keys and to the groups.
+        """
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Key %s</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2><span class="refentrytitle">%s</span></h2>
+
+<div class="refsynopsisdiv">
+<h2>Synopsis</h2>
+<pre class="synopsis">
+%s
+</pre>
+</div>
+
+<div class="refsynopsisdiv">
+<h2>Related Keys</h2>
+<pre class="synopsis">
+%s
+</pre>
+</div>
+
+<div class="refsynopsisdiv">
+<h2>Groups</h2>
+<pre class="synopsis">
+%s
+</pre>
+</div>
+
+
+</body>
+"""     % (item.name(), self.createNavigator(), item.name(), 
+           self.escape(item.description()), self.relatedKeys(item), self.groups(item))
+
+    def createGroupsSite(self, doc):
+        """
+        Create the Group Overview site
+        """
+
+        groups = ""
+        sorted_groups = doc.groups()
+        sorted_groups.sort()
+        for group in sorted_groups:
+            groups += """<a href="group%s.html">%s</a><br>""" % (group, group)
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Group overview</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2>Available Groups</h2>
+%s
+</body>
+""" % (self.createNavigator(), groups)
+
+    def createIndex(self):
+        """
+        Create the index file
+        """
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Bitbake Documentation</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2>Documentation Entrance</h2>
+<a href="groups.html">All available groups</a><br>
+<a href="keys.html">All available keys</a><br>
+</body>
+""" % self.createNavigator()
+
+    def createKeysSite(self, doc):
+        """
+        Create Overview of all avilable keys
+        """
+        keys = ""
+        sorted_keys = doc.doc_keys()
+        sorted_keys.sort()
+        for key in sorted_keys:
+            keys += """<a href="key%s.html">%s</a><br>""" % (key, key)
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Key overview</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<h2>Available Keys</h2>
+%s
+</body>
+""" % (self.createNavigator(), keys)
+
+    def createGroupSite(self,gr, items):
+        """
+        Create a site for a group:
+        Group the name of the group, items contain the name of the keys
+        inside this group
+        """
+        groups = ""
+        for group in items:
+            groups += """<a href="key%s.html">%s</a><br>""" % (group.name(), group.name())
+
+        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html><head><title>Group %s</title></head>
+<link rel="stylesheet" href="style.css" type="text/css">
+<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
+%s
+<div class="refsynopsisdiv">
+<h2>Keys in Group %s</h2>
+<pre class="synopsis">
+%s
+</pre>
+</div>
+</body>
+""" % (gr, self.createNavigator(), gr, groups)
+
+
+
+    def createCSS(self):
+        """
+        Create the CSS file
+        """
+        return """.synopsis, .classsynopsis
+{
+  background: #eeeeee;
+  border: solid 1px #aaaaaa;
+  padding: 0.5em;
+}
+.programlisting
+{
+  background: #eeeeff;
+  border: solid 1px #aaaaff;
+  padding: 0.5em;
+}
+.variablelist
+{
+  padding: 4px;
+  margin-left: 3em;
+}
+.variablelist td:first-child
+{
+  vertical-align: top;
+}
+table.navigation
+{
+  background: #ffeeee;
+  border: solid 1px #ffaaaa;
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+.navigation a
+{
+  color: #770000;
+}
+.navigation a:visited
+{
+  color: #550000;
+}
+.navigation .title
+{
+  font-size: 200%;
+}
+div.refnamediv
+{
+  margin-top: 2em;
+}
+div.gallery-float
+{
+  float: left;
+  padding: 10px;
+}
+div.gallery-float img
+{
+  border-style: none;
+}
+div.gallery-spacer
+{
+  clear: both;
+}
+a
+{
+  text-decoration: none;
+}
+a:hover
+{
+  text-decoration: underline;
+  color: #FF0000;
+}
+"""
+
+
+
+class DocumentationItem:
+    """
+    A class to hold information about a configuration
+    item. It contains the key name, description, a list of related names,
+    and the group this item is contained in.
+    """
+
+    def __init__(self):
+        self._groups  = []
+        self._related = []
+        self._name    = ""
+        self._desc    = ""
+
+    def groups(self):
+        return self._groups
+
+    def name(self):
+        return self._name
+
+    def description(self):
+        return self._desc
+
+    def related(self):
+        return self._related
+
+    def setName(self, name):
+        self._name = name
+
+    def setDescription(self, desc):
+        self._desc = desc
+
+    def addGroup(self, group):
+        self._groups.append(group)
+
+    def addRelation(self,relation):
+        self._related.append(relation)
+
+    def sort(self):
+        self._related.sort()
+        self._groups.sort()
+
+
+class Documentation:
+    """
+    Holds the documentation... with mappings from key to items...
+    """
+
+    def __init__(self):
+        self.__keys   = {}
+        self.__groups = {}
+
+    def insert_doc_item(self, item):
+        """
+        Insert the Doc Item into the internal list
+        of representation
+        """
+        item.sort()
+        self.__keys[item.name()] = item
+
+        for group in item.groups():
+            if not group in self.__groups:
+                self.__groups[group] = []
+            self.__groups[group].append(item)
+            self.__groups[group].sort()
+
+
+    def doc_item(self, key):
+        """
+        Return the DocumentationInstance describing the key
+        """
+        try:
+            return self.__keys[key]
+        except KeyError:
+            return None
+
+    def doc_keys(self):
+        """
+        Return the documented KEYS (names)
+        """
+        return self.__keys.keys()
+
+    def groups(self):
+        """
+        Return the names of available groups
+        """
+        return self.__groups.keys()
+
+    def group_content(self,group_name):
+        """
+        Return a list of keys/names that are in a specefic
+        group or the empty list
+        """
+        try:
+            return self.__groups[group_name]
+        except KeyError:
+            return []
+
+
+def parse_cmdline(args):
+    """
+    Parse the CMD line and return the result as a n-tuple
+    """
+
+    parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__,__version__))
+    usage  = """%prog [options]
+
+Create a set of html pages (documentation) for a bitbake.conf....
+"""
+
+    # Add the needed options
+    parser.add_option( "-c", "--config", help = "Use the specified configuration file as source",
+                       action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") )
+
+    parser.add_option( "-o", "--output", help = "Output directory for html files",
+                       action = "store", dest = "output", default = "html/" )
+
+    parser.add_option( "-D",  "--debug", help = "Increase the debug level",
+                       action = "count", dest = "debug", default = 0 )
+
+    parser.add_option( "-v","--verbose", help = "output more chit-char to the terminal",
+                       action = "store_true", dest = "verbose", default = False )
+
+    options, args = parser.parse_args( sys.argv )
+
+    if options.debug:
+        bb.debug_level = options.debug
+
+    return options.config, options.output
+
+def main():
+    """
+    The main Method
+    """
+
+    (config_file,output_dir) = parse_cmdline( sys.argv )
+
+    # right to let us load the file now
+    try:
+        documentation = bb.parse.handle( config_file, bb.data.init() )
+    except IOError:
+        bb.fatal( "Unable to open %s" % config_file )
+    except bb.parse.ParseError:
+        bb.fatal( "Unable to parse %s" % config_file )
+
+
+    # Assuming we've the file loaded now, we will initialize the 'tree'
+    doc = Documentation()
+
+    # defined states
+    state_begin = 0
+    state_see   = 1
+    state_group = 2
+
+    for key in bb.data.keys(documentation):
+        data   = bb.data.getVarFlag(key, "doc", documentation)
+        if not data:
+            continue
+
+        # The Documentation now starts
+        doc_ins = DocumentationItem()
+        doc_ins.setName(key)
+
+
+        tokens = data.split(' ')
+        state = state_begin
+        string= ""
+        for token in tokens:
+            token = token.strip(',')
+
+            if not state == state_see and token == "@see":
+                state = state_see
+                continue
+            elif not state == state_group and token  == "@group":
+                state = state_group
+                continue
+
+            if state == state_begin:
+                string += " %s" % token
+            elif state == state_see:
+                doc_ins.addRelation(token)
+            elif state == state_group:
+                doc_ins.addGroup(token)
+
+        # set the description
+        doc_ins.setDescription(string)
+        doc.insert_doc_item(doc_ins)
+
+    # let us create the HTML now
+    bb.mkdirhier(output_dir)
+    os.chdir(output_dir)
+
+    # Let us create the sites now. We do it in the following order
+    # Start with the index.html. It will point to sites explaining all
+    # keys and groups
+    html_slave = HTMLFormatter()
+
+    f = file('style.css', 'w')
+    print >> f, html_slave.createCSS()
+
+    f = file('index.html', 'w')
+    print >> f, html_slave.createIndex()
+
+    f = file('groups.html', 'w')
+    print >> f, html_slave.createGroupsSite(doc)
+
+    f = file('keys.html', 'w')
+    print >> f, html_slave.createKeysSite(doc)
+
+    # now for each group create the site
+    for group in doc.groups():
+        f = file('group%s.html' % group, 'w')
+        print >> f, html_slave.createGroupSite(group, doc.group_content(group))
+
+    # now for the keys
+    for key in doc.doc_keys():
+        f = file('key%s.html' % doc.doc_item(key).name(), 'w')
+        print >> f, html_slave.createKeySite(doc.doc_item(key))
+
+
+if __name__ == "__main__":
+    main()