Changeset b1c2cb4 in rtems-source-builder

Timestamp:

07/14/19 04:03:06 (4 years ago)

Author:

Chris Johns <chrisj@…>

Branches:

5, master

Children:

831ae05

Parents:

f68f0d9

git-author:

Chris Johns <chrisj@…> (07/14/19 04:03:06)

git-committer:

Chris Johns <chrisj@…> (07/21/19 11:09:34)

Message:

sb/setbuilder: Support staging of build set build before installing.

• Staging lets build sets use previously built build sets.

• Fix the rtems-bsp builds to support staging.

Files:

• rtems/config/5/rtems-libbsd.bset (modified) (1 diff)
• rtems/config/rtems-bsp.cfg (modified) (6 diffs)
• rtems/config/tools/rtems-libbsd-common.cfg (modified) (1 diff)
• source-builder/defaults.mc (modified) (2 diffs)
• source-builder/sb/setbuilder.py (modified) (23 diffs)
• source-builder/sb/shell.py (added)

rtems/config/5/rtems-libbsd.bset

@@ -4 +4 @@
 
 #
-# Build the kernel.
+# Build the BSD library for RTEMS.
 #
 tools/rtems-libbsd-5

rtems/config/rtems-bsp.cfg

@@ -25 +25 @@
 # and BSPs. Only after the source to download.
 #
-%if %{_dry_run} && %{defined with_download}
+%if %{_dry_run}
  %log BSP configuration errors ignored
  %define rtems_bsp_error 0
@@ -37 +37 @@
 #
 %if %{defined rtems_host}
- %define _host %{rtems_host}
+ %{triplet _host %{rtems_host}}
 %endif
 %if %{rtems_bsp_error}
@@ -56 +56 @@
 
 #
-# If no tools or RTEMS provided use the prefix.
+# If no tools or RTEMS provided use the prefix. If staging use the staging
+# root.
 #
 %ifn %{defined with_tools}
- %define with_tools %{_prefix}
- %define rtems_waf_tools %{nil}
+ %if %{install_mode} == staging
+  %define with_tools %{stagingroot}
+  %define rtems_waf_tools --rtems-tools=%{with_tools}
+ %else
+  %define with_tools %{_prefix}
+  %define rtems_waf_tools %{nil}
+ %endif
 %else
  %define rtems_waf_tools --rtems-tools=%{with_tools}
@@ -66 +72 @@
 
 %ifn %{defined with_rtems}
- %define with_rtems %{_prefix}
- %define rtems_waf_rtems %{nil}
+ %if %{install_mode} == staging
+  %define with_rtems %{stagingroot}
+  %define rtems_waf_rtems --rtems=%{with_rtems}
+ %else
+  %define with_rtems %{_prefix}
+  %define rtems_waf_rtems %{nil}
+ %endif
 %else
  %define rtems_waf_rtems --rtems=%{with_rtems}
@@ -112 +123 @@
 #
 %define rtems_bsp_cc      %{_host}-gcc
+%define rtems_bsp_cxx     %{_host}-g++
 %define rtems_bsp_incpath %{with_rtems}/%{_host}/%{rtems_bsp}/include
 %define rtems_bsp_libpath %{with_rtems}/%{_host}/%{rtems_bsp}/lib
@@ -161 +173 @@
 # Map to names used for cross compiling.
 #
+%define host_cc       %{rtems_bsp_cc}
+%define host_cxx      %{rtems_bsp_cxx}
 %define host_includes -I%{_includedir}
 %define host_cflags   %{rtems_bsp_cflags}

rtems/config/tools/rtems-libbsd-common.cfg

@@ -63 +63 @@
     --prefix=%{_prefix} \
     %{rtems_waf_tools} \
-    %{rtems_waf_rtems}
+    %{rtems_waf_rtems} \
     --rtems-bsp=%{rtems_bsp_arch_bsp}
 

source-builder/defaults.mc

@@ -75 +75 @@
 build_includes:      none,    convert,  ''
 build_libs:          none,    convert,  ''
+
+#
+# Build and staging paths.
+#
+buildroot:           dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}'
+buildcxcroot:        dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}-cxc'
+buildxcroot:         dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}-xx'
+stagingroot:         dir,     none,     '%{_tmppath}/sb-%{_uid}-staging'
+
+#
+# Install mode can be installing or staging. Defaults to installing.
+#
+install_mode:        none,    none,     'installing'
 
 # Extra path a platform can override.
@@ -97 +110 @@
 _tmproot:            dir,     none,     '%{_tmppath}/sb-%{_uid}/%{_bset_tmp}'
 _tmpcxcroot:         dir,     none,     '%{_tmppath}/sb-%{_uid}-cxc/%{_bset_tmp}'
-buildroot:           dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}'
-buildcxcroot:        dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}-cxc'
-buildxcroot:         dir,     none,     '%{_tmppath}/%{buildname}-%{_uid}-xx'
 _datadir:            dir,     none,     '%{_prefix}/share'
 _defaultdocdir:      dir,     none,     '%{_prefix}/share/doc'

source-builder/sb/setbuilder.py

@@ -42 +42 @@
     import path
     import reports
+    import shell
     import sources
     import version
@@ -51 +52 @@
     sys.exit(1)
 
+def macro_expand(macros, _str):
+    cstr = None
+    while cstr != _str:
+        cstr = _str
+        _str = macros.expand(_str)
+        _str = shell.expand(macros, _str)
+    return _str
+
 class log_capture(object):
     def __init__(self):
@@ -72 +81 @@
 
     def __init__(self, bset, _configs, opts, macros = None):
-        log.trace('_bset: %s: init' % (bset))
+        log.trace('_bset:   : %s: init' % (bset))
         self.configs = _configs
         self.opts = opts
@@ -79 +88 @@
         else:
             self.macros = copy.copy(macros)
-        log.trace('_bset: %s: macro defaults' % (bset))
+        log.trace('_bset:   : %s: macro defaults' % (bset))
         log.trace(str(self.macros))
         self.bset = bset
-        _target = self.macros.expand('%{_target}')
+        _target = macro_expand(self.macros, '%{_target}')
         if len(_target):
             pkg_prefix = _target
         else:
-            pkg_prefix = self.macros.expand('%{_host}')
+            pkg_prefix = macro_expand(self.macros, '%{_host}')
         self.bset_pkg = '%s-%s-set' % (pkg_prefix, self.bset)
         self.mail_header = ''
@@ -153 +162 @@
                 raise error.general('invalid report format: %s' % (format))
             buildroot = _build.config.abspath('%{buildroot}')
-            prefix = _build.macros.expand('%{_prefix}')
+            prefix = macro_expand(_build.macros, '%{_prefix}')
             name = _build.main_package().name() + ext
             log.notice('reporting: %s -> %s' % (_config, name))
@@ -183 +192 @@
         what = '%s -> %s' % \
             (os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
-        log.trace('_bset: %s: collecting: %s' % (self.bset, what))
+        log.trace('_bset:   : %s: collecting: %s' % (self.bset, what))
         self.copy(src, dst)
 
-    def install(self, name, buildroot, prefix):
-        dst = prefix
-        src = path.join(buildroot, prefix)
-        log.notice('installing: %s -> %s' % (name, path.host(dst)))
+    def install(self, mode, name, src, dst):
+        log.notice('%s: %s -> %s' % (mode, name, path.host(dst)))
         self.copy(src, dst)
 
+    def install_mode(self):
+        return macro_expand(self.macros, '%{install_mode}')
+
+    def installing(self):
+        return self.install_mode() == 'installing'
+
+    def staging(self):
+        return not self.installing()
+
     def canadian_cross(self, _build):
-        log.trace('_bset: Cxc for build machine: _build => _host')
+        log.trace('_bset:   : Cxc for build machine: _build => _host')
         macros_to_copy = [('%{_host}',        '%{_build}'),
                           ('%{_host_alias}',  '%{_build_alias}'),
@@ -205 +221 @@
         cxc_macros = _build.copy_init_macros()
         for m in macros_to_copy:
-            log.trace('_bset: Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
+            log.trace('_bset:   : Cxc: %s <= %s' % (m[0], cxc_macros[m[1]]))
             cxc_macros[m[0]] = cxc_macros[m[1]]
         _build.set_macros(cxc_macros)
@@ -230 +246 @@
            and not _build.macros.get('%{_disable_packaging}'):
             path.mkdir(tardir)
-            tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (_build.main_package().name())))
+            tar = path.join(tardir,
+                            _build.config.expand('%s.tar.bz2' % \
+                                                 (_build.main_package().name())))
             log.notice('tarball: %s' % (os.path.relpath(path.host(tar))))
             if not self.opts.dry_run():
                 tmproot = _build.config.expand('%{_tmproot}')
                 cmd = _build.config.expand('"cd ' + tmproot + \
-                                               ' && %{__tar} -cf - . | %{__bzip2} > ' + tar + '"')
+                                           ' && %{__tar} -cf - . | %{__bzip2} > ' + \
+                                           tar + '"')
                 _build.run(cmd, shell_opts = '-c', cwd = tmproot)
 
@@ -250 +269 @@
 
         if not path.exists(bsetname):
-            for cp in self.macros.expand('%{_configdir}').split(':'):
+            for cp in macro_expand(self.macros, '%{_configdir}').split(':'):
                 configdir = path.abspath(cp)
                 bsetname = path.join(configdir, bset)
@@ -259 +278 @@
                 raise error.general('no build set file found: %s' % (bset))
         try:
-            log.trace('_bset: %s: open: %s' % (self.bset, bsetname))
+            log.trace('_bset:   : %s: open: %s' % (self.bset, bsetname))
             bset = open(path.host(bsetname), 'r')
         except IOError as err:
@@ -273 +292 @@
                 if len(l) == 0:
                     continue
-                log.trace('_bset: %s: %03d: %s' % (self.bset, lc, l))
+                log.trace('_bset:   : %s: %03d: %s' % (self.bset, lc, l))
                 ls = l.split()
                 if ls[0][-1] == ':' and ls[0][:-1] == 'package':
@@ -289 +308 @@
                     elif ls[0] == '%undefine':
                         if len(ls) > 2:
-                            raise error.general('%s:%d: %undefine requires just the name' % \
-                                                    (self.bset, lc))
+                            raise error.general('%s:%d: %undefine requires ' \
+                                                'just the name' % (self.bset, lc))
                         self.macros.undefine(ls[1].strip())
                     elif ls[0] == '%include':
@@ -302 +321 @@
                     c = build.find_config(l, self.configs)
                     if c is None:
-                        raise error.general('%s:%d: cannot find file: %s' % (self.bset, lc, l))
+                        raise error.general('%s:%d: cannot find file: %s' % (self.bset,
+                                                                             lc, l))
                     configs += [c]
         except:
@@ -321 +341 @@
             configs = [self.bset]
         else:
-            exbset = self.macros.expand(self.bset)
+            exbset = macro_expand(self.macros, self.bset)
             self.macros['_bset'] = exbset
-            self.macros['_bset_tmp'] = build.short_name(exbset)
+            bset_tmp = build.short_name(exbset)
+            if bset_tmp.endswith('.bset'):
+                bset_tmp = bset_tmp[:-5]
+            self.macros['_bset_tmp'] = bset_tmp
             root, ext = path.splitext(exbset)
             if exbset.endswith('.bset'):
@@ -341 +364 @@
             mail['output'].clear()
 
-        log.trace('_bset: %s: make' % (self.bset))
+        log.trace('_bset: %2d: %s: make' % (nesting_count, self.bset))
         log.notice('Build Set: %s' % (self.bset))
 
@@ -351 +374 @@
         start = datetime.datetime.now()
 
-        mail_report = False
-        have_errors = False
+        mail_report  = False
+        have_errors  = False
 
         if mail:
             mail['output'].clear()
 
+        #
+        # If this is the outter most buildset it's files installed. Nested
+        # build sets staged their installed file. The staged files are install
+        # when the outtter most build finishes.
+        #
+        if nesting_count != 1:
+            if self.installing():
+                self.macros['install_mode'] = 'staging'
+                #
+                # Prepend staging areas, bin directory tothe
+                # path. Lets the later package depend on the eailier
+                # ones.
+                #
+                pathprepend = ['%{stagingroot}/bin'] + \
+                    macro_expand(self.macros, '%{_pathprepend}').split(':')
+                pathprepend = [pp for pp in pathprepend if len(pp)]
+                if len(pathprepend) == 1:
+                    self.macros['_pathprepend'] = pathprepend[0]
+                else:
+                    self.macros['_pathprepend'] = ':'.join(pathprepend)
+
+        #
+        # Only the outter build set can have staging to install. Get the staging
+        # root via the config because it could require a valid config.
+        #
+        have_staging = False
+
         try:
             configs = self.load()
 
-            log.trace('_bset: %s: configs: %s'  % (self.bset, ','.join(configs)))
+            log.trace('_bset: %2d: %s: configs: %s'  % (nesting_count,
+                                                        self.bset, ', '.join(configs)))
 
             sizes_valid = False
@@ -375 +426 @@
                     macros = copy.copy(self.macros)
                     if configs[s].endswith('.bset'):
-                        log.trace('_bset: == %2d %s' % (nesting_count + 1, '=' * 75))
+                        log.trace('_bset: %2d: %s %s' % (nesting_count,
+                                                         configs[s],
+                                                         '=' * (74 - len(configs[s]))))
                         bs = buildset(configs[s], self.configs, opts, macros)
                         bs.build(deps, nesting_count, mail)
+                        if self.installing():
+                            have_staging = True
                         del bs
                     elif configs[s].endswith('.cfg'):
                         if mail:
                             mail_report = True
-                        log.trace('_bset: -- %2d %s' % (nesting_count + 1, '-' * 75))
+                        log.trace('_bset: %2d: %s %s' % (nesting_count,
+                                                         configs[s],
+                                                         '=' * (74 - len(configs[s]))))
                         try:
                             b = build.build(configs[s],
@@ -413 +470 @@
                         # Dump post build macros.
                         #
-                        log.trace('_bset: macros post-build')
+                        log.trace('_bset:   : macros post-build')
                         log.trace(str(b.macros))
                     else:
@@ -437 +494 @@
                         raise
             #
-            # Installing ...
-            #
-            log.trace('_bset: installing: deps:%r no-install:%r' % \
-                      (deps is None, self.opts.no_install()))
-            if deps is None \
-               and not self.opts.no_install() \
-               and not have_errors:
+            # Installing or staging ...
+            #
+            log.trace('_bset: %2d: %s: deps:%r no-install:%r' % \
+                      (nesting_count, self.install_mode(),
+                       deps is None, self.opts.no_install()))
+            log.trace('_bset: %2d: %s: builds: %s' % \
+                      (nesting_count, self.install_mode(),
+                       ', '.join([b.name() for b in builds])))
+            if deps is None and not have_errors:
                 for b in builds:
-                    log.trace('_bset: installing: %r' % b.installable())
+                    log.trace('_bset:   : %s: %r' % (self.install_mode(),
+                                                     b.installable()))
                     if b.installable():
-                        self.install(b.name(),
-                                     b.config.expand('%{buildroot}'),
-                                     b.config.expand('%{_prefix}'))
+                        prefix = b.config.expand('%{_prefix}')
+                        buildroot = path.join(b.config.expand('%{buildroot}'), prefix)
+                        if self.staging():
+                            prefix = b.config.expand('%{stagingroot}')
+                        self.install(self.install_mode(), b.name(), buildroot, prefix)
+
             #
             # Sizes ...
@@ -470 +533 @@
                     size_patches += path.get_size(p)
                 size_total = size_sources + size_patches + size_installed
-                build_max_size_human = build.humanize_number(size_build_max + size_installed, 'B')
+                build_max_size_human = build.humanize_number(size_build_max +
+                                                             size_installed, 'B')
                 build_total_size_human = build.humanize_number(size_total, 'B')
                 build_sources_size_human = build.humanize_number(size_sources, 'B')
@@ -500 +564 @@
             for b in builds:
                 del b
+
+            #
+            # If builds have been staged install into the finaly prefix.
+            #
+            if have_staging and not self.opts.no_install() and not have_errors:
+                log.trace('_bset: %2d: install staging' % (nesting_count))
+                stagingroot = macro_expand(self.macros, '%{stagingroot}')
+                prefix = macro_expand(self.macros, '%{_prefix}')
+                self.install(self.install_mode(), self.bset, stagingroot, prefix)
+                staging_size = path.get_size(stagingroot)
+                if not self.opts.no_clean() or self.opts.always_clean():
+                    log.notice('clean staging: %s' % (self.bset))
+                    log.trace('cleanup: %s' % (stagingroot))
+                    self.rmdir(stagingroot)
+                log.notice('Staging Size: %s' % (build.humanize_number(staging_size)))
         except error.general as gerr:
             if not build_error:
@@ -587 +666 @@
                 mail['to'] = to_addr[1]
             else:
-                mail['to'] = opts.defaults.expand('%{_mail_tools_to}')
+                mail['to'] = macro_expand(opts.defaults, '%{_mail_tools_to}')
             mail['from'] = mail['mail'].from_address()
         log.notice('RTEMS Source Builder - Set Builder, %s' % (version.str()))
@@ -607 +686 @@
             deps = None
         if not list_bset_cfg_files(opts, configs):
-            prefix = opts.defaults.expand('%{_prefix}')
+            prefix = macro_expand(opts.defaults, '%{_prefix}')
             if opts.canadian_cross():
                 opts.disable_install()