source: rtems-libbsd/builder.py

6-freebsd-12
Last change on this file was a8c55b6, checked in by Sebastian Huber <sebastian.huber@…>, on 02/16/22 at 14:27:41

builder.py: Fix addition of plain text files
  • Property mode set to 100755
File size: 41.3 KB
Line 
1# SPDX-License-Identifier: BSD-2-Clause
2"""Manage the libbsd build configuration data.
3"""
4
5# Copyright (c) 2015, 2020 Chris Johns <chrisj@rtems.org>. All rights reserved.
6#
7# Copyright (c) 2009, 2017 embedded brains GmbH.  All rights reserved.
8#
9#   embedded brains GmbH
10#   Dornierstr. 4
11#   82178 Puchheim
12#   Germany
13#   <info@embedded-brains.de>
14#
15# Redistribution and use in source and binary forms, with or without
16# modification, are permitted provided that the following conditions
17# are met:
18# 1. Redistributions of source code must retain the above copyright
19#    notice, this list of conditions and the following disclaimer.
20# 2. Redistributions in binary form must reproduce the above copyright
21#    notice, this list of conditions and the following disclaimer in the
22#    documentation and/or other materials provided with the distribution.
23#
24# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34# POSSIBILITY OF SUCH DAMAGE.
35
36from __future__ import print_function
37
38import codecs
39import copy
40import difflib
41import os
42import re
43import sys
44
45try:
46    import configparser
47except ImportError:
48    import ConfigParser as configparser
49
50#
51# Global controls.
52#
53LIBBSD_DIR = "."
54FreeBSD_DIR = "freebsd-org"
55verboseLevel = 0
56isDryRun = False
57isDiffMode = False
58filesProcessedCount = 0
59filesProcessed = []
60filesTotal = 0
61filesTotalLines = 0
62filesTotalInserts = 0
63filesTotalDeletes = 0
64diffDetails = {}
65
66verboseInfo = 1
67verboseDetail = 2
68verboseMoreDetail = 3
69verboseDebug = 4
70
71BUILDSET_DIR = "buildset"
72BUILDSET_DEFAULT = "buildset/default.ini"
73
74
75def verbose(level=verboseInfo):
76    return verboseLevel >= level
77
78
79def changedFileSummary(statsReport=False):
80
81    global filesTotal, filesTotalLines, filesTotalInserts, filesTotalDeletes
82
83    if isDiffMode == False:
84        if verbose():
85            print('%d file(s) were changed:' % (filesProcessedCount))
86            for f in sorted(filesProcessed):
87                print(' %s' % (f))
88        else:
89            print('%d file(s) were changed.' % (filesProcessedCount))
90    if statsReport:
91        print('Stats Report:')
92        transparent = filesTotal - len(diffDetails)
93        changes = filesTotalInserts + filesTotalDeletes
94        opacity = (float(changes) / (filesTotalLines + changes)) * 100.0
95        print(' Total File(s):%d  Unchanged:%d (%.1f%%)  Changed:%d' \
96              '   Opacity:%5.1f%% Lines:%d Edits:%d (+):%d (-):%d'  % \
97              (filesTotal, transparent, (float(transparent) / filesTotal) * 100.0, len(diffDetails), \
98               opacity, filesTotalLines, changes, filesTotalInserts, filesTotalDeletes))
99        #
100        # Sort by opacity.
101        #
102        ordered_diffs = sorted(diffDetails.items(),
103                               key=lambda diff: diff[1].opacity,
104                               reverse=True)
105        for f in ordered_diffs:
106            print('  %s' % (diffDetails[f[0]].status()))
107
108
109def readFile(name):
110    try:
111        contents = codecs.open(name,
112                               mode='r',
113                               encoding='utf-8',
114                               errors='ignore').read()
115    except UnicodeDecodeError as ude:
116        print('error: reading: %s: %s' % (name, ude))
117        sys.exit(1)
118    return contents
119
120
121def writeFile(name, contents):
122    path = os.path.dirname(name)
123    if not os.path.exists(path):
124        try:
125            os.makedirs(path)
126        except OSError as oe:
127            print('error: cannot create directory: %s: %s' % (path, oe))
128            sys.exit(1)
129    try:
130        codecs.open(name, mode='w', encoding='utf-8',
131                    errors='ignore').write(contents)
132    except UnicodeDecodeError as ude:
133        print('error: write: %s: %s' % (name, ude))
134        sys.exit(1)
135
136
137#
138# A builder error.
139#
140class error(Exception):
141    """Base class for exceptions."""
142    def __init__(self, msg):
143        self.msg = 'error: %s' % (msg)
144
145    def set_output(self, msg):
146        self.msg = msg
147
148    def __str__(self):
149        return self.msg
150
151
152#
153# Diff Record
154#
155class diffRecord:
156    def __init__(self, src, dst, orig, diff, inserts, deletes):
157        self.src = src
158        self.dst = dst
159        self.orig = orig
160        self.diff = diff
161        self.lines = len(orig)
162        self.inserts = inserts
163        self.deletes = deletes
164        self.changes = inserts + deletes
165        self.opacity = (float(self.changes) /
166                        (self.lines + self.changes)) * 100.0
167
168    def __repr__(self):
169        return self.src
170
171    def status(self):
172        return 'opacity:%5.1f%% edits:%4d (+):%-4d (-):%-4d %s' % \
173            (self.opacity, self.changes, self.inserts, self.deletes, self.src)
174
175
176#
177# This stuff needs to move to libbsd.py.
178#
179
180
181# Move target dependent files under a machine directory
182def mapCPUDependentPath(path):
183    return path.replace("include/", "include/machine/")
184
185
186def fixIncludes(data):
187    data = re.sub('#include <sys/resource.h>',
188                  '#include <rtems/bsd/sys/resource.h>', data)
189    data = re.sub('#include <sys/unistd.h>',
190                  '#include <rtems/bsd/sys/unistd.h>', data)
191    return data
192
193
194# revert fixing the include paths inside a C or .h file
195def revertFixIncludes(data):
196    data = re.sub('#include <rtems/bsd/', '#include <', data)
197    data = re.sub('#include <util.h>', '#include <rtems/bsd/util.h>', data)
198    data = re.sub('#include <bsd.h>', '#include <rtems/bsd/bsd.h>', data)
199    data = re.sub('#include <zerocopy.h>', '#include <rtems/bsd/zerocopy.h>',
200                  data)
201    data = re.sub('#include <modules.h>', '#include <rtems/bsd/modules.h>',
202                  data)
203    return data
204
205
206# fix include paths inside a C or .h file
207def fixLocalIncludes(data):
208    data = re.sub('#include "opt_([^"]*)"',
209                  '#include <rtems/bsd/local/opt_\\1>', data)
210    data = re.sub('#include "([^"]*)_if.h"',
211                  '#include <rtems/bsd/local/\\1_if.h>', data)
212    data = re.sub('#include "miidevs([^"]*)"',
213                  '#include <rtems/bsd/local/miidevs\\1>', data)
214    data = re.sub('#include "usbdevs([^"]*)"',
215                  '#include <rtems/bsd/local/usbdevs\\1>', data)
216    return data
217
218
219# revert fixing the include paths inside a C or .h file
220def revertFixLocalIncludes(data):
221    data = re.sub('#include <rtems/bsd/local/([^>]*)>', '#include "\\1"', data)
222    return data
223
224def assertNothing(path):
225    pass
226
227def assertHeaderFile(path):
228    if path[-2] != '.' or path[-1] != 'h':
229        print("*** " + path + " does not end in .h")
230        print("*** Move it to a C source file list")
231        sys.exit(2)
232
233
234def assertSourceFile(path):
235    if path[-2:] != '.c' and path[-2:] != '.S' and path[-3:] != '.cc':
236        print("*** " + path + " does not end in .c, .cc or .S")
237        print("*** Move it to a header file list")
238        sys.exit(2)
239
240
241def assertHeaderOrSourceFile(path):
242    if path[-2] != '.' or (path[-1] != 'h' and path[-1] != 'c'):
243        print("*** " + path + " does not end in .h or .c")
244        print("*** Move it to another list")
245        sys.exit(2)
246
247
248def diffSource(dstLines, srcLines, src, dst):
249    global filesTotal, filesTotalLines, filesTotalInserts, filesTotalDeletes
250    #
251    # Diff, note there is no line termination on each string.  Expand the
252    # generator to list because the generator is not reusable.
253    #
254    diff = list(
255        difflib.unified_diff(dstLines,
256                             srcLines,
257                             fromfile=src,
258                             tofile=dst,
259                             n=5,
260                             lineterm=''))
261    inserts = 0
262    deletes = 0
263    if len(diff) > 0:
264        if src in diffDetails and \
265           diffDetails[src].dst != dst and diffDetails[src].diff != diff:
266            raise error('repeated diff of file different: src:%s dst:%s' %
267                        (src, dst))
268        for l in diff:
269            if l[0] == '-':
270                deletes += 1
271            elif l[0] == '+':
272                inserts += 1
273        diffDetails[src] = diffRecord(src, dst, srcLines, diff, inserts,
274                                      deletes)
275
276    #
277    # Count the total files, lines and the level of changes.
278    #
279    filesTotal += 1
280    filesTotalLines += len(srcLines)
281    filesTotalInserts += inserts
282    filesTotalDeletes += deletes
283
284    return diff
285
286
287#
288# Converters provide a way to alter the various types of code. The conversion
289# process filters a file as it is copies from the source path to the
290# destination path. Specialised versions are provided for different types of
291# source.
292#
293class Converter(object):
294    def convert(self,
295                src,
296                dst,
297                hasSource=True,
298                sourceFilter=None,
299                srcContents=None):
300
301        global filesProcessed, filesProcessedCount
302
303        if verbose(verboseDebug):
304            print("convert: filter:%s: %s -> %s" % \
305                  (['yes', 'no'][sourceFilter is None], src, dst))
306
307        #
308        # If there is no source raise an error if we expect source else print a
309        # warning and do not try and convert.
310        #
311        if srcContents is None:
312            if not os.path.exists(src):
313                if hasSource:
314                    raise error('source not found: %s' % (src))
315                else:
316                    print('warning: no source: %s' % (src))
317                    return
318
319            #
320            # Files read as a single string if not passed in.
321            #
322            srcContents = readFile(src)
323
324        if os.path.exists(dst):
325            dstContents = readFile(dst)
326        else:
327            print('warning: no destination: %s' % (dst))
328            dstContents = ''
329
330        #
331        # Filter the source.
332        #
333        if sourceFilter is not None:
334            srcContents = sourceFilter(srcContents)
335
336        #
337        # Split into a list of lines.
338        #
339        srcLines = srcContents.split(os.linesep)
340        dstLines = dstContents.split(os.linesep)
341
342        if verbose(verboseDebug):
343            print('Unified diff: %s (lines:%d)' % (src, len(srcLines)))
344
345        #
346        # Diff, note there is no line termination on each string.
347        #
348        diff = diffSource(dstLines, srcLines, src, dst)
349
350        #
351        # The diff list is empty if the files are the same.
352        #
353        if len(diff) > 0:
354
355            if verbose(verboseDebug):
356                print('Unified diff length: %d' % len(diff))
357
358            filesProcessed += [dst]
359            filesProcessedCount += 1
360            if isDiffMode == False:
361                if verbose(verboseDetail):
362                    print("UPDATE: %s -> %s" % (src, dst))
363                if isDryRun == False:
364                    writeFile(dst, srcContents)
365            else:
366                print("diff -u %s %s" % (src, dst))
367                for l in diff:
368                    print(l)
369
370
371class NoConverter(Converter):
372    def convert(self, src, dst, hasSource=True, sourceFilter=None):
373        return '/* EMPTY */\n'
374
375
376class FromFreeBSDToRTEMSHeaderConverter(Converter):
377    def sourceFilter(self, data):
378        data = fixLocalIncludes(data)
379        data = fixIncludes(data)
380        return data
381
382    def convert(self, src, dst):
383        sconverter = super(FromFreeBSDToRTEMSHeaderConverter, self)
384        sconverter.convert(src, dst, sourceFilter=self.sourceFilter)
385
386
387class FromFreeBSDToRTEMSUserSpaceHeaderConverter(Converter):
388    def sourceFilter(self, data):
389        data = fixIncludes(data)
390        return data
391
392    def convert(self, src, dst):
393        sconverter = super(FromFreeBSDToRTEMSUserSpaceHeaderConverter, self)
394        sconverter.convert(src, dst, sourceFilter=self.sourceFilter)
395
396
397class FromFreeBSDToRTEMSSourceConverter(Converter):
398    def sourceFilter(self, data):
399        data = fixLocalIncludes(data)
400        data = fixIncludes(data)
401        data = '#include <machine/rtems-bsd-kernel-space.h>\n\n' + data
402        return data
403
404    def convert(self, src, dst):
405        sconverter = super(FromFreeBSDToRTEMSSourceConverter, self)
406        sconverter.convert(src, dst, sourceFilter=self.sourceFilter)
407
408
409class FromFreeBSDToRTEMSUserSpaceSourceConverter(Converter):
410    def sourceFilter(self, data):
411        data = fixIncludes(data)
412        data = '#include <machine/rtems-bsd-user-space.h>\n\n' + data
413        return data
414
415    def convert(self, src, dst):
416        sconverter = super(FromFreeBSDToRTEMSUserSpaceSourceConverter, self)
417        sconverter.convert(src, dst, sourceFilter=self.sourceFilter)
418
419
420class FromRTEMSToFreeBSDHeaderConverter(Converter):
421    def sourceFilter(self, data):
422        data = revertFixLocalIncludes(data)
423        data = revertFixIncludes(data)
424        return data
425
426    def convert(self, src, dst):
427        sconverter = super(FromRTEMSToFreeBSDHeaderConverter, self)
428        sconverter.convert(src,
429                           dst,
430                           hasSource=False,
431                           sourceFilter=self.sourceFilter)
432
433
434class FromRTEMSToFreeBSDSourceConverter(Converter):
435    def sourceFilter(self, data):
436        data = re.sub('#include <machine/rtems-bsd-kernel-space.h>\n\n', '',
437                      data)
438        data = re.sub('#include <machine/rtems-bsd-user-space.h>\n\n', '',
439                      data)
440        data = revertFixLocalIncludes(data)
441        data = revertFixIncludes(data)
442        return data
443
444    def convert(self, src, dst):
445        sconverter = super(FromRTEMSToFreeBSDSourceConverter, self)
446        sconverter.convert(src,
447                           dst,
448                           hasSource=False,
449                           sourceFilter=self.sourceFilter)
450
451
452#
453# Compose a path based for the various parts of the source tree.
454#
455class PathComposer(object):
456    def composeOriginPath(self, path):
457        return path
458
459    def composeLibBSDPath(self, path, prefix):
460        return os.path.join(prefix, path)
461
462
463class FreeBSDPathComposer(PathComposer):
464    def composeOriginPath(self, path):
465        return os.path.join(FreeBSD_DIR, path)
466
467    def composeLibBSDPath(self, path, prefix):
468        return os.path.join(prefix, 'freebsd', path)
469
470
471class RTEMSPathComposer(PathComposer):
472    def composeOriginPath(self, path):
473        return path
474
475    def composeLibBSDPath(self, path, prefix):
476        return os.path.join(prefix, 'rtemsbsd', path)
477
478
479class LinuxPathComposer(PathComposer):
480    def composeOriginPath(self, path):
481        return path
482
483    def composeLibBSDPath(self, path, prefix):
484        return os.path.join(prefix, 'linux', path)
485
486
487class CPUDependentFreeBSDPathComposer(FreeBSDPathComposer):
488    def composeLibBSDPath(self, path, prefix):
489        path = super(CPUDependentFreeBSDPathComposer,
490                     self).composeLibBSDPath(path, prefix)
491        path = mapCPUDependentPath(path)
492        return path
493
494
495class CPUDependentRTEMSPathComposer(RTEMSPathComposer):
496    def composeLibBSDPath(self, path, prefix):
497        path = super(CPUDependentRTEMSPathComposer,
498                     self).composeLibBSDPath(path, prefix)
499        path = mapCPUDependentPath(path)
500        return path
501
502
503class CPUDependentLinuxPathComposer(LinuxPathComposer):
504    def composeLibBSDPath(self, path, prefix):
505        path = super(CPUDependentLinuxPathComposer,
506                     self).composeLibBSDPath(path, prefix)
507        path = mapCPUDependentPath(path)
508        return path
509
510
511class TargetSourceCPUDependentPathComposer(CPUDependentFreeBSDPathComposer):
512    def __init__(self, targetCPU, sourceCPU):
513        self.targetCPU = targetCPU
514        self.sourceCPU = sourceCPU
515
516    def composeLibBSDPath(self, path, prefix):
517        path = super(TargetSourceCPUDependentPathComposer,
518                     self).composeLibBSDPath(path, prefix)
519        path = path.replace(self.sourceCPU, self.targetCPU)
520        return path
521
522
523class BuildSystemComposer(object):
524    def __init__(self, includes=None):
525        if type(includes) is not list:
526            self.includes = [includes]
527        else:
528            self.includes = includes
529
530    def __str__(self):
531        return ''
532
533    def getIncludes(self):
534        if None in self.includes:
535            incs = []
536        else:
537            incs = self.includes
538        return incs
539
540    def compose(self, path):
541        """A None result means there is nothing to build."""
542        return None
543
544    @staticmethod
545    def filesAsDefines(files):
546        define_keys = ''
547        for f in files:
548            f = f.upper()
549            for c in '\/-.':
550                f = f.replace(c, '_')
551            define_keys += ' ' + f
552        return define_keys.strip()
553
554    @staticmethod
555    def cflagsIncludes(cflags, includes):
556        if type(cflags) is not list:
557            if cflags is not None:
558                _cflags = cflags.split(' ')
559            else:
560                _cflags = [None]
561        else:
562            _cflags = cflags
563        if type(includes) is not list:
564            _includes = [includes]
565        else:
566            _includes = includes
567        return _cflags, _includes
568
569
570class SourceFileBuildComposer(BuildSystemComposer):
571    def __init__(self, cflags="default", includes=None):
572        self.cflags, self.includes = self.cflagsIncludes(cflags, includes)
573
574    def __str__(self):
575        return 'SF: ' + ' '.join(self.getFlags())
576
577    def compose(self, path):
578        flags = self.getFlags()
579        return ['sources', flags, ('default', None)], \
580            [path], self.cflags, self.includes
581
582    def getFlags(self):
583        return self.cflags + self.getIncludes()
584
585
586class SourceFileIfHeaderComposer(SourceFileBuildComposer):
587    def __init__(self, headers, cflags="default", includes=None):
588        if headers is not list:
589            headers = [headers]
590        self.headers = headers
591        super(SourceFileIfHeaderComposer, self).__init__(cflags=cflags,
592                                                         includes=includes)
593
594    def __str__(self):
595        return 'SFIH:' + ' '.join(self.getFlags()) + \
596            ' ' + self.filesAsDefines(self.headers)
597
598    def compose(self, path):
599        r = SourceFileBuildComposer.compose(self, path)
600        r[0][2] = (self.filesAsDefines(self.headers), self.headers)
601        return r
602
603
604class TestFragementComposer(BuildSystemComposer):
605    def __init__(self,
606                 testName,
607                 fileFragments,
608                 configTest=None,
609                 runTest=True,
610                 netTest=False,
611                 extraLibs=[],
612                 modules=[]):
613        self.testName = testName
614        self.fileFragments = fileFragments
615        self.configTest = configTest
616        self.runTest = runTest
617        self.netTest = netTest
618        self.extraLibs = extraLibs
619        self.modules = modules
620
621    def __str__(self):
622        return 'TEST: ' + self.testName
623
624    def compose(self, path):
625        return ['tests', self.testName, ('default', None)], {
626            'configTest': self.configTest,
627            'files': self.fileFragments,
628            'run': self.runTest,
629            'net': self.netTest,
630            'libs': self.extraLibs,
631            'modules': self.modules,
632        }
633
634
635class TestIfHeaderComposer(TestFragementComposer):
636    def __init__(self,
637                 testName,
638                 headers,
639                 fileFragments,
640                 runTest=True,
641                 netTest=False,
642                 extraLibs=[],
643                 modules=[]):
644        if headers is not list:
645            headers = [headers]
646        self.headers = headers
647        super(TestIfHeaderComposer, self).__init__(testName,
648                                                   fileFragments,
649                                                   'header',
650                                                   runTest=runTest,
651                                                   netTest=netTest,
652                                                   extraLibs=extraLibs,
653                                                   modules=modules)
654
655    def compose(self, path):
656        r = TestFragementComposer.compose(self, path)
657        r[0][2] = (self.filesAsDefines(self.headers), self.headers)
658        return r
659
660
661class TestIfLibraryComposer(TestFragementComposer):
662    def __init__(self,
663                 testName,
664                 libraries,
665                 fileFragments,
666                 runTest=True,
667                 netTest=False,
668                 extraLibs=[],
669                 modules=[]):
670        if libraries is not list:
671            libraries = [libraries]
672        self.libraries = libraries
673        super(TestIfLibraryComposer, self).__init__(testName,
674                                                    fileFragments,
675                                                    'library',
676                                                    runTest=runTest,
677                                                    netTest=netTest,
678                                                    extraLibs=extraLibs,
679                                                    modules=modules)
680
681    def compose(self, path):
682        r = TestFragementComposer.compose(self, path)
683        r[0][2] = (self.filesAsDefines(self.libraries), self.libraries)
684        return r
685
686
687class KVMSymbolsBuildComposer(BuildSystemComposer):
688    def compose(self, path):
689        return ['KVMSymbols', 'files', ('default', None)], \
690            [path], self.includes
691
692
693class RPCGENBuildComposer(BuildSystemComposer):
694    def compose(self, path):
695        return ['RPCGen', 'files', ('default', None)], \
696            [path]
697
698
699class RouteKeywordsBuildComposer(BuildSystemComposer):
700    def compose(self, path):
701        return ['RouteKeywords', 'files', ('default', None)], \
702            [path]
703
704
705class LexBuildComposer(BuildSystemComposer):
706    def __init__(self, sym, dep, cflags=None, includes=None, build=True):
707        self.sym = sym
708        self.dep = dep
709        self.cflags, self.includes = self.cflagsIncludes(cflags, includes)
710        self.build = build
711
712    def compose(self, path):
713        d = {
714            'file': path,
715            'sym': self.sym,
716            'dep': self.dep,
717            'build': self.build
718        }
719        if None not in self.cflags:
720            d['cflags'] = self.cflags
721        if None not in self.includes:
722            d['includes'] = self.includes
723        return ['lex', path, ('default', None)], d
724
725
726class YaccBuildComposer(BuildSystemComposer):
727    def __init__(self, sym, header, cflags=None, includes=None, build=True):
728        self.sym = sym
729        self.header = header
730        self.cflags, self.includes = self.cflagsIncludes(cflags, includes)
731        self.build = build
732
733    def compose(self, path):
734        d = {
735            'file': path,
736            'sym': self.sym,
737            'header': self.header,
738            'build': self.build
739        }
740        if None not in self.cflags:
741            d['cflags'] = self.cflags
742        if None not in self.includes:
743            d['includes'] = self.includes
744        return ['yacc', path, ('default', None)], d
745
746
747class File(object):
748    '''A file of source we move backwards and forwards and build.'''
749    def __init__(self, space, path, pathComposer, forwardConverter,
750                 reverseConverter, buildSystemComposer):
751        if verbose(verboseMoreDetail):
752            print("FILE: %-6s %-50s F:%-45s R:%-45s" % \
753                  (space, path,
754                   forwardConverter.__class__.__name__,
755                   reverseConverter.__class__.__name__))
756        self.space = space
757        self.path = path
758        self.pathComposer = pathComposer
759        self.originPath = self.pathComposer.composeOriginPath(self.path)
760        self.libbsdPath = self.pathComposer.composeLibBSDPath(
761            self.path, LIBBSD_DIR)
762        self.forwardConverter = forwardConverter
763        self.reverseConverter = reverseConverter
764        self.buildSystemComposer = buildSystemComposer
765
766    def __str__(self):
767        out = self.space[0].upper() + ' ' + self.path
768        bsc = str(self.buildSystemComposer)
769        if len(bsc) > 0:
770            out += ' (' + bsc + ')'
771        return out
772
773    def __eq__(self, other):
774        state = self.space == other.space
775        state = state and (self.path == self.path)
776        state = state and (self.pathComposer == self.pathComposer)
777        state = state and (self.originPath == self.originPath)
778        state = state and (self.forwardConverter == self.forwardConverter)
779        state = state and (self.self.reverseConverter
780                           == self.self.reverseConverter)
781        state = state and (self.buildSystemComposer
782                           == self.buildSystemComposer)
783        return state
784
785    def processSource(self, forward):
786        if forward:
787            if verbose(verboseDetail):
788                print("process source: %s => %s" %
789                      (self.originPath, self.libbsdPath))
790            self.forwardConverter.convert(self.originPath, self.libbsdPath)
791        else:
792            if verbose(verboseDetail):
793                print("process source: %s => %s converter:%s" % \
794                      (self.libbsdPath, self.originPath,
795                       self.reverseConverter.__class__.__name__))
796            self.reverseConverter.convert(self.libbsdPath, self.originPath)
797
798    def getFragment(self):
799        return self.buildSystemComposer.compose(
800            self.pathComposer.composeLibBSDPath(self.path, ''))
801
802    def getPath(self):
803        return self.path
804
805    def getSpace(self):
806        return self.space
807
808
809class Module(object):
810    '''Logical group of related files we can perform actions on'''
811    def __init__(self, manager, name, enabled=True):
812        self.manager = manager
813        self.name = name
814        self.files = []
815        self.cpuDependentSourceFiles = {}
816        self.dependencies = []
817
818    def __str__(self):
819        out = [self.name + ':']
820        if len(self.dependencies) > 0:
821            out += [' Deps: ' + str(len(self.dependencies))]
822            out += ['  ' + type(d).__name__ for d in self.dependencies]
823        if len(self.files) > 0:
824            counts = {}
825            for f in self.files:
826                space = f.getSpace()
827                if space not in counts:
828                    counts[space] = 0
829                counts[space] += 1
830            count_str = ''
831            for space in sorted(counts.keys()):
832                count_str += '%s:%d ' % (space[0].upper(), counts[space])
833            count_str = count_str[:-1]
834            out += [' Files: %d (%s)' % (len(self.files), count_str)]
835            out += ['  ' + str(f) for f in self.files]
836        if len(self.cpuDependentSourceFiles) > 0:
837            out += [' CPU Dep: ' + str(len(self.cpuDependentSourceFiles))]
838            for cpu in self.cpuDependentSourceFiles:
839                out += ['  ' + cpu + ':']
840                out += [
841                    '   ' + str(f) for f in self.cpuDependentSourceFiles[cpu]
842                ]
843        return os.linesep.join(out)
844
845    def initCPUDependencies(self, cpu):
846        if cpu not in self.cpuDependentSourceFiles:
847            self.cpuDependentSourceFiles[cpu] = []
848
849    def getName(self):
850        return self.name
851
852    def getFiles(self):
853        return (f for f in self.files)
854
855    def processSource(self, direction):
856        if verbose(verboseDetail):
857            print("process module: %s" % (self.name))
858        for f in self.files:
859            f.processSource(direction)
860        for cpu, files in self.cpuDependentSourceFiles.items():
861            for f in files:
862                f.processSource(direction)
863
864    def addFile(self, f):
865        if not isinstance(f, File):
866            raise TypeError('invalid type for addFiles: %s' % (type(f)))
867        self.files += [f]
868
869    def addFiles(self,
870                 space,
871                 newFiles,
872                 pathComposer,
873                 forwardConverter,
874                 reverseConverter,
875                 assertFile,
876                 buildSystemComposer=BuildSystemComposer()):
877        files = []
878        for newFile in newFiles:
879            assertFile(newFile)
880            files += [
881                File(space, newFile, pathComposer, forwardConverter,
882                     reverseConverter, buildSystemComposer)
883            ]
884        return files
885
886    def addPlainTextFiles(self, files):
887        self.files += self.addFiles('user', files,
888                                    FreeBSDPathComposer(), Converter(),
889                                    Converter(), assertNothing)
890
891    def addKernelSpaceHeaderFiles(self, files):
892        self.files += self.addFiles('kernel', files, FreeBSDPathComposer(),
893                                    FromFreeBSDToRTEMSHeaderConverter(),
894                                    FromRTEMSToFreeBSDHeaderConverter(),
895                                    assertHeaderOrSourceFile)
896
897    def addUserSpaceHeaderFiles(self, files):
898        self.files += self.addFiles(
899            'user', files, FreeBSDPathComposer(),
900            FromFreeBSDToRTEMSUserSpaceHeaderConverter(),
901            FromRTEMSToFreeBSDHeaderConverter(), assertHeaderFile)
902
903    def addRTEMSHeaderFiles(self, files):
904        self.files += self.addFiles('user', files, RTEMSPathComposer(),
905                                    NoConverter(), NoConverter(),
906                                    assertHeaderFile)
907
908    def addLinuxHeaderFiles(self, files):
909        self.files += self.addFiles('kernel', files, PathComposer(),
910                                    NoConverter(), NoConverter(),
911                                    assertHeaderFile)
912
913    def addCPUDependentFreeBSDHeaderFiles(self, files):
914        self.files += self.addFiles('kernel', files,
915                                    CPUDependentFreeBSDPathComposer(),
916                                    FromFreeBSDToRTEMSHeaderConverter(),
917                                    FromRTEMSToFreeBSDHeaderConverter(),
918                                    assertHeaderFile)
919
920    def addCPUDependentLinuxHeaderFiles(self, files):
921        self.files += self.addFiles('kernel', files,
922                                    CPUDependentLinuxPathComposer(),
923                                    NoConverter(), NoConverter(),
924                                    assertHeaderFile)
925
926    def addTargetSourceCPUDependentHeaderFiles(self, targetCPUs, sourceCPU,
927                                               files):
928        for cpu in targetCPUs:
929            self.files += self.addFiles(
930                'kernel', files,
931                TargetSourceCPUDependentPathComposer(cpu, sourceCPU),
932                FromFreeBSDToRTEMSHeaderConverter(), NoConverter(),
933                assertHeaderFile)
934
935    def addSourceFiles(self, files, sourceFileBuildComposer):
936        self.files += self.addFiles('user',
937                                    files, PathComposer(), NoConverter(),
938                                    NoConverter(), assertSourceFile,
939                                    sourceFileBuildComposer)
940
941    def addKernelSpaceSourceFiles(self, files, sourceFileBuildComposer):
942        self.files += self.addFiles('kernel', files, FreeBSDPathComposer(),
943                                    FromFreeBSDToRTEMSSourceConverter(),
944                                    FromRTEMSToFreeBSDSourceConverter(),
945                                    assertSourceFile, sourceFileBuildComposer)
946
947    def addUserSpaceSourceFiles(self, files, sourceFileBuildComposer):
948        self.files += self.addFiles(
949            'user', files, FreeBSDPathComposer(),
950            FromFreeBSDToRTEMSUserSpaceSourceConverter(),
951            FromRTEMSToFreeBSDSourceConverter(), assertSourceFile,
952            sourceFileBuildComposer)
953
954    def addRTEMSKernelSourceFiles(self, files, sourceFileBuildComposer):
955        self.files += self.addFiles('kernel', files, RTEMSPathComposer(),
956                                    NoConverter(), NoConverter(),
957                                    assertSourceFile, sourceFileBuildComposer)
958
959    def addRTEMSUserSourceFiles(self, files, sourceFileBuildComposer):
960        self.files += self.addFiles('user', files, RTEMSPathComposer(),
961                                    NoConverter(), NoConverter(),
962                                    assertSourceFile, sourceFileBuildComposer)
963
964    def addLinuxSourceFiles(self, files, sourceFileBuildComposer):
965        self.files += self.addFiles('kernel', files, PathComposer(),
966                                    NoConverter(), NoConverter(),
967                                    assertSourceFile, sourceFileBuildComposer)
968
969    def addCPUDependentFreeBSDSourceFiles(self, cpus, files,
970                                          sourceFileBuildComposer):
971        for cpu in cpus:
972            self.initCPUDependencies(cpu)
973            self.cpuDependentSourceFiles[cpu] += \
974                self.addFiles(
975                    'kernel', files,
976                    CPUDependentFreeBSDPathComposer(), FromFreeBSDToRTEMSSourceConverter(),
977                    FromRTEMSToFreeBSDSourceConverter(), assertSourceFile,
978                    sourceFileBuildComposer)
979
980    def addCPUDependentRTEMSSourceFiles(self, cpus, files,
981                                        sourceFileBuildComposer):
982        for cpu in cpus:
983            self.initCPUDependencies(cpu)
984            self.cpuDependentSourceFiles[cpu] += \
985                self.addFiles('kernel', files,
986                              CPUDependentRTEMSPathComposer(), NoConverter(),
987                              NoConverter(), assertSourceFile,
988                              sourceFileBuildComposer)
989
990    def addCPUDependentLinuxSourceFiles(self, cpus, files,
991                                        sourceFileBuildComposer):
992        for cpu in cpus:
993            self.initCPUDependencies(cpu)
994            self.cpuDependentSourceFiles[cpu] += \
995                self.addFiles('kernel', files,
996                              CPUDependentLinuxPathComposer(), NoConverter(),
997                              NoConverter(), assertSourceFile,
998                              sourceFileBuildComposer)
999
1000    def addTest(self, testFragementComposer):
1001        self.files += [
1002            File('user', testFragementComposer.testName, PathComposer(),
1003                 NoConverter(), NoConverter(), testFragementComposer)
1004        ]
1005
1006    def addDependency(self, dep):
1007        if not isinstance(dep, str):
1008            raise TypeError('dependencies are a string: %s' % (self.name))
1009        self.dependencies += [dep]
1010
1011
1012class ModuleManager(object):
1013    '''A manager for a collection of modules.'''
1014    def __init__(self):
1015        self.modules = {}
1016        self.generator = {}
1017        self.configuration = {}
1018        self.setGenerators()
1019
1020    def __getitem__(self, key):
1021        if key not in self.modules:
1022            raise KeyError('module %s not found' % (key))
1023        return self.modules[key]
1024
1025    def __str__(self):
1026        out = ['Modules: ' + str(len(self.modules)), '']
1027        for m in sorted(self.modules):
1028            out += [str(self.modules[m]), '']
1029        return os.linesep.join(out)
1030
1031    def _loadIni(self, ini_file):
1032        if not os.path.exists(ini_file):
1033            raise FileNotFoundError('file not found: %s' % (ini_file))
1034        ini = configparser.ConfigParser()
1035        ini.read(ini_file)
1036        if not ini.has_section('general'):
1037            raise Exception(
1038                "'{}' is missing a general section.".format(ini_file))
1039        if not ini.has_option('general', 'name'):
1040            raise Exception("'{}' is missing a general/name.".format(ini_file))
1041        if ini.has_option('general', 'extends'):
1042            extends = ini.get('general', 'extends')
1043            extendfile = None
1044            basepath = os.path.dirname(ini_file)
1045            if os.path.isfile(os.path.join(basepath, extends)):
1046                extendfile = os.path.join(basepath, extends)
1047            elif os.path.isfile(os.path.join(BUILDSET_DIR, extends)):
1048                extendfile = os.path.join(BUILDSET_DIR, extends)
1049            else:
1050                raise Exception(
1051                    "'{}': Invalid file given for general/extends:'{}'".format(
1052                        ini_file, extends))
1053            base = self._loadIni(extendfile)
1054            for s in ini.sections():
1055                if not base.has_section(s):
1056                    base.add_section(s)
1057                for o in ini.options(s):
1058                    val = ini.get(s, o)
1059                    base.set(s, o, val)
1060            ini = base
1061        return ini
1062
1063    def _checkDependencies(self):
1064        enabled_modules = self.getEnabledModules()
1065        if 'tests' in enabled_modules:
1066            enabled_modules.remove('tests')
1067        for mod in enabled_modules:
1068            if mod not in self.modules:
1069                raise KeyError('enabled module not found: %s' % (mod))
1070            for dep in self.modules[mod].dependencies:
1071                if dep not in self.modules:
1072                    print(type(dep))
1073                    raise KeyError('dependent module not found: %s' % (dep))
1074                if dep not in enabled_modules:
1075                    raise Exception('module "%s" dependency "%s" not enabled' %
1076                                    (mod, dep))
1077
1078    def getAllModules(self):
1079        if 'modules' in self.configuration:
1080            return sorted(self.configuration['modules'])
1081        return []
1082
1083    def getEnabledModules(self):
1084        if 'modules-enabled' in self.configuration:
1085            return sorted(self.configuration['modules-enabled'])
1086        return []
1087
1088    def addModule(self, module):
1089        name = module.name
1090        if name in self.modules:
1091            raise KeyError('module already added: %' % (name))
1092        self.modules[name] = module
1093        if 'modules' not in self.configuration:
1094            self.configuration['modules'] = []
1095        if 'modules-enabled' not in self.configuration:
1096            self.configuration['modules-enabled'] = []
1097        self.configuration['modules'] += [name]
1098        self.configuration['modules-enabled'] += [name]
1099        self.configuration['modules'].sort()
1100        self.configuration['modules-enabled'].sort
1101
1102    def processSource(self, direction):
1103        if verbose(verboseDetail):
1104            print("process modules:")
1105        for m in sorted(self.modules):
1106            self.modules[m].processSource(direction)
1107
1108    def setConfiguration(self, config):
1109        self.configuration = copy.deepcopy(config)
1110
1111    def getConfiguration(self):
1112        return copy.deepcopy(self.configuration)
1113
1114    def generateBuild(self, only_enabled=True):
1115        modules_to_process = self.getEnabledModules()
1116        # Used for copy between FreeBSD and RTEMS
1117        if only_enabled == False:
1118            modules_to_process = self.getAllModules()
1119        for m in modules_to_process:
1120            if m not in self.modules:
1121                raise KeyError('enabled module not registered: %s' % (m))
1122            self.modules[m].generate()
1123        self._checkDependencies()
1124
1125    def duplicateCheck(self):
1126        dups = []
1127        modules_to_check = sorted(self.getAllModules(), reverse=True)
1128        while len(modules_to_check) > 1:
1129            mod = modules_to_check.pop()
1130            for m in modules_to_check:
1131                if m not in self.modules:
1132                    raise KeyError('enabled module not registered: %s' % (m))
1133                for fmod in self.modules[mod].getFiles():
1134                    for fm in self.modules[m].getFiles():
1135                        if fmod.getPath() == fm.getPath():
1136                            dups += [(m, mod, fm.getPath(), fm.getSpace())]
1137        return dups
1138
1139    def loadConfig(self, config=BUILDSET_DEFAULT):
1140        if 'name' in self.configuration:
1141            raise KeyError('configuration already loaded: %s (%s)' % \
1142                           (self.configuration['name'], config))
1143        ini = self._loadIni(config)
1144        self.configuration['name'] = ini.get('general', 'name')
1145        self.configuration['modules-enabled'] = []
1146        mods = []
1147        if ini.has_section('modules'):
1148            mods = ini.options('modules')
1149        for mod in mods:
1150            if ini.getboolean('modules', mod):
1151                self.configuration['modules-enabled'].append(mod)
1152
1153    def getName(self):
1154        if 'name' not in self.configuration:
1155            raise KeyError('configuration not loaded')
1156        return self.configuration['name']
1157
1158    def setGenerators(self):
1159        self.generator['convert'] = Converter
1160        self.generator['no-convert'] = NoConverter
1161        self.generator[
1162            'from-FreeBSD-to-RTEMS-UserSpaceSourceConverter'] = \
1163                FromFreeBSDToRTEMSUserSpaceSourceConverter
1164        self.generator[
1165            'from-RTEMS-To-FreeBSD-SourceConverter'] = FromRTEMSToFreeBSDSourceConverter
1166        self.generator['buildSystemComposer'] = BuildSystemComposer
1167
1168        self.generator['file'] = File
1169
1170        self.generator['path'] = PathComposer
1171        self.generator['freebsd-path'] = FreeBSDPathComposer
1172        self.generator['rtems-path'] = RTEMSPathComposer
1173        self.generator['cpu-path'] = CPUDependentFreeBSDPathComposer
1174        self.generator[
1175            'target-src-cpu--path'] = TargetSourceCPUDependentPathComposer
1176
1177        self.generator['source'] = SourceFileBuildComposer
1178        self.generator['test'] = TestFragementComposer
1179        self.generator['kvm-symbols'] = KVMSymbolsBuildComposer
1180        self.generator['rpc-gen'] = RPCGENBuildComposer
1181        self.generator['route-keywords'] = RouteKeywordsBuildComposer
1182        self.generator['lex'] = LexBuildComposer
1183        self.generator['yacc'] = YaccBuildComposer
1184
1185        self.generator['source-if-header'] = SourceFileIfHeaderComposer
1186        self.generator['test-if-header'] = TestIfHeaderComposer
1187        self.generator['test-if-library'] = TestIfLibraryComposer
Note: See TracBrowser for help on using the repository browser.