Changeset f1fcdba in rtems-libbsd


Ignore:
Timestamp:
Apr 27, 2016, 2:03:17 AM (4 years ago)
Author:
Chris Johns <chrisj@…>
Branches:
afaeccc05a556f6aa25ba044a7e49d6aa634a59e, freebsd-9.3, master
Children:
3c7fdb2
Parents:
5b93cc8
git-author:
Chris Johns <chrisj@…> (04/27/16 02:03:17)
git-committer:
Chris Johns <chrisj@…> (04/27/16 02:08:23)
Message:

waf: Refector the builder to work with Python3 and UTF-8 source files.

Python 3 requires better UTF-8 handling of files and FreeBSD has UTF-8
characters in some files.

Refactor builder.py to clean up the code and remove the need to have
a temporary file. Update other scripts to use the new code.

Files:
3 edited

Legend:

Unmodified
Added
Removed
  • builder.py

    r5b93cc8 rf1fcdba  
    4444import filecmp
    4545import difflib
     46import codecs
    4647
    4748#
     
    5051RTEMS_DIR = "."
    5152FreeBSD_DIR = "freebsd-org"
    52 isVerbose = False
     53verboseLevel = 0
    5354isDryRun = False
    5455isDiffMode = False
     
    5657filesProcessed = []
    5758
     59verboseInfo = 1
     60verboseDetail = 2
     61verboseMoreDetail = 3
     62verboseDebug = 4
     63
     64def verbose(level = verboseInfo):
     65    return verboseLevel >= level
     66
    5867def changedFileSummary():
    5968    if isDiffMode == False:
    6069        print('%d file(s) were changed:' % (filesProcessedCount))
    61         for f in sorted(filesProcessed):
    62             print(' %s' % (f))
    63 
     70        if verbose():
     71            for f in sorted(filesProcessed):
     72                print(' %s' % (f))
     73
     74def read_file(name):
     75    try:
     76        contents = codecs.open(name, mode = 'r', encoding = 'utf-8', errors = 'ignore').read()
     77    except UnicodeDecodeError as ude:
     78        print('error: reading: %s: %s' % (name, ude))
     79        sys.exit(1)
     80    return contents
     81
     82def write_file(name, contents):
     83    path = os.path.dirname(name)
     84    if not os.path.exists(path):
     85        try:
     86            os.makedirs(path)
     87        except OSError as oe:
     88            print('error: cannot create directory: %s: %s' % (dst_path, oe))
     89            sys.exit(1)
     90    try:
     91        codecs.open(name, mode = 'w',  encoding = 'utf-8', errors = 'ignore').write(contents)
     92    except UnicodeDecodeError as ude:
     93        print('error: write: %s: %s' % (name, ude))
     94        sys.exit(1)
     95
     96#
     97# A builder error.
     98#
    6499class error(Exception):
    65100    """Base class for exceptions."""
    66     def __init(self, msg):
     101    def __init__(self, msg):
    67102        self.msg = 'error: %s' % (msg)
    68103    def set_output(self, msg):
     
    71106        return self.msg
    72107
     108#
     109# This stuff needs to move to libbsd.py.
     110#
    73111def common_flags():
    74112    return ['-O2',
     
    137175            ('mDNSResponder/mDNSPosix',       'mDNSPosix.h',       '')]
    138176
    139 # compare and process file only if different
    140 #  + copy or diff depending on execution mode
    141 def processIfDifferent(new, old, src):
    142 
    143     global filesProcessedCount
    144     global filesProcessed
    145     global isVerbose, isDryRun, isEarlyExit
    146 
    147     if not os.path.exists(old) or \
    148        filecmp.cmp(new, old, shallow = False) == False:
    149         filesProcessed += [old]
    150         filesProcessedCount += 1
    151         if isDiffMode == False:
    152             if isVerbose == True:
    153                 print("Move " + src + " to " + old)
    154             if isDryRun == False:
    155                 shutil.move(new, old)
    156         else:
    157             if isVerbose == True:
    158                 print("Diff %s => %s" % (src, new))
    159             old_contents = open(old).readlines()
    160             new_contents = open(new).readlines()
    161             for line in \
    162                 difflib.unified_diff(old_contents, new_contents,
    163                                      fromfile = src, tofile = new, n = 5):
    164                 sys.stdout.write(line)
    165 
    166177# Move target dependent files under a machine directory
    167178def mapCPUDependentPath(path):
     
    169180
    170181def fixIncludes(data):
    171     data = re.sub('#include <sys/lock.h>', '#include <rtems/bsd/sys/lock.h>', data)
    172     data = re.sub('#include <sys/time.h>', '#include <rtems/bsd/sys/time.h>', data)
    173     data = re.sub('#include <sys/cpuset.h>', '#include <rtems/bsd/sys/cpuset.h>', data)
    174     data = re.sub('#include <sys/errno.h>', '#include <rtems/bsd/sys/errno.h>', data)
    175     data = re.sub('#include <sys/param.h>', '#include <rtems/bsd/sys/param.h>', data)
    176     data = re.sub('#include <sys/types.h>', '#include <rtems/bsd/sys/types.h>', data)
     182    data = re.sub('#include <sys/lock.h>',     '#include <rtems/bsd/sys/lock.h>', data)
     183    data = re.sub('#include <sys/time.h>',     '#include <rtems/bsd/sys/time.h>', data)
     184    data = re.sub('#include <sys/cpuset.h>',   '#include <rtems/bsd/sys/cpuset.h>', data)
     185    data = re.sub('#include <sys/errno.h>',    '#include <rtems/bsd/sys/errno.h>', data)
     186    data = re.sub('#include <sys/param.h>',    '#include <rtems/bsd/sys/param.h>', data)
     187    data = re.sub('#include <sys/types.h>',    '#include <rtems/bsd/sys/types.h>', data)
    177188    data = re.sub('#include <sys/resource.h>', '#include <rtems/bsd/sys/resource.h>', data)
    178     data = re.sub('#include <sys/unistd.h>', '#include <rtems/bsd/sys/unistd.h>', data)
    179     data = re.sub('#include <sys/_types.h>', '#include <rtems/bsd/sys/_types.h>', data)
     189    data = re.sub('#include <sys/unistd.h>',   '#include <rtems/bsd/sys/unistd.h>', data)
     190    data = re.sub('#include <sys/_types.h>',   '#include <rtems/bsd/sys/_types.h>', data)
    180191    return data
    181192
    182193# revert fixing the include paths inside a C or .h file
    183194def revertFixIncludes(data):
    184     data = re.sub('#include <rtems/bsd/', '#include <', data)
    185     data = re.sub('#include <util.h>', '#include <rtems/bsd/util.h>', data)
    186     data = re.sub('#include <bsd.h>', '#include <rtems/bsd/bsd.h>', data)
     195    data = re.sub('#include <rtems/bsd/',  '#include <', data)
     196    data = re.sub('#include <util.h>',     '#include <rtems/bsd/util.h>', data)
     197    data = re.sub('#include <bsd.h>',      '#include <rtems/bsd/bsd.h>', data)
    187198    data = re.sub('#include <zerocopy.h>', '#include <rtems/bsd/zerocopy.h>', data)
    188199    return data
     
    190201# fix include paths inside a C or .h file
    191202def fixLocalIncludes(data):
    192     data = re.sub('#include "opt_([^"]*)"', '#include <rtems/bsd/local/opt_\\1>', data)
    193     data = re.sub('#include "([^"]*)_if.h"', '#include <rtems/bsd/local/\\1_if.h>', data)
     203    data = re.sub('#include "opt_([^"]*)"',    '#include <rtems/bsd/local/opt_\\1>', data)
     204    data = re.sub('#include "([^"]*)_if.h"',   '#include <rtems/bsd/local/\\1_if.h>', data)
    194205    data = re.sub('#include "miidevs([^"]*)"', '#include <rtems/bsd/local/miidevs\\1>', data)
    195206    data = re.sub('#include "usbdevs([^"]*)"', '#include <rtems/bsd/local/usbdevs\\1>', data)
     
    213224        sys.exit(2)
    214225
     226#
     227# Converters provide a way to alter the various types of code. The conversion
     228# process filters a file as it is copies from the source path to the
     229# destination path. Specialised versions are provided for different types of
     230# source.
     231#
    215232class Converter(object):
    216     def convert(self, src):
    217         return open(src).read()
    218 
    219     def isConvertible(self):
    220         return True
     233
     234    def convert(self, src, dst, has_source = True, source_filter = None, src_contents = None):
     235
     236        global filesProcessed, filesProcessedCount
     237
     238        if verbose(verboseDebug):
     239            print("convert: filter:%s: %s -> %s" % \
     240                  (['yes', 'no'][source_filter is None], src, dst))
     241
     242        #
     243        # If there is no source raise an error if we expect source else print a
     244        # warning and do not try and convert.
     245        #
     246        if src_contents is None:
     247            if not os.path.exists(src):
     248                if has_source:
     249                    raise error('source not found: %s' % (src))
     250                else:
     251                    print('warning: no source: %s' % (src))
     252                    return
     253
     254            #
     255            # Files read as a single string if not passed in.
     256            #
     257            src_contents = read_file(src)
     258
     259        if os.path.exists(dst):
     260            dst_contents = read_file(dst)
     261        else:
     262            print('warning: no destination: %s' % (dst))
     263            dst_contents = ''
     264
     265        #
     266        # Filter the source.
     267        #
     268        if source_filter is not None:
     269            src_contents = source_filter(src_contents)
     270
     271        #
     272        # Split into a list of lines.
     273        #
     274        src_lines = src_contents.split(os.linesep)
     275        dst_lines = dst_contents.split(os.linesep)
     276
     277        if verbose(verboseDebug):
     278            print('Unified diff: %s (lines:%d)' % (src, len(src_lines)))
     279
     280        #
     281        # Diff, note there is no line termination on each string.  Expand the
     282        # generator to list because the generator is not reusable.
     283        #
     284        diff = list(difflib.unified_diff(dst_lines,
     285                                         src_lines,
     286                                         fromfile = src,
     287                                         tofile = dst,
     288                                         n = 5,
     289                                         lineterm = ''))
     290
     291        #
     292        # The diff list is empty if the files are the same.
     293        #
     294        if len(diff) > 0:
     295
     296            if verbose(verboseDebug):
     297                print('Unified diff length: %d' % len(diff))
     298
     299            filesProcessed += [dst]
     300            filesProcessedCount += 1
     301            if isDiffMode == False:
     302                if verbose(verboseDetail):
     303                    print("UPDATE: %s -> %s" % (src, dst))
     304                if isDryRun == False:
     305                    write_file(dst, src_contents)
     306            else:
     307                print("diff -u %s %s" % (src, dst))
     308                for l in diff:
     309                    print(l)
    221310
    222311class NoConverter(Converter):
    223     def convert(self, src):
    224         raise
    225 
    226     def isConvertible(self):
    227         return False
    228 
    229 class EmptyConverter(Converter):
    230     def convert(self, src):
     312    def convert(self, src, dst, has_source = True, source_filter = None):
    231313        return '/* EMPTY */\n'
    232314
    233315class FromFreeBSDToRTEMSHeaderConverter(Converter):
    234     def convert(self, src):
    235         data = super(FromFreeBSDToRTEMSHeaderConverter, self).convert(src)
     316    def source_filter(self, data):
    236317        data = fixLocalIncludes(data)
    237318        data = fixIncludes(data)
    238319        return data
    239320
     321    def convert(self, src, dst):
     322        sconverter = super(FromFreeBSDToRTEMSHeaderConverter, self)
     323        sconverter.convert(src, dst, source_filter = self.source_filter)
     324
    240325class FromFreeBSDToRTEMSUserSpaceHeaderConverter(Converter):
    241     def convert(self, src):
    242         data = super(FromFreeBSDToRTEMSUserSpaceHeaderConverter, self).convert(src)
     326    def source_filter(self, data):
    243327        data = fixIncludes(data)
    244328        return data
    245329
     330    def convert(self, src, dst):
     331        sconverter = super(FromFreeBSDToRTEMSUserSpaceHeaderConverter, self)
     332        sconverter.convert(src, dst, source_filter = self.source_filter)
     333
    246334class FromFreeBSDToRTEMSSourceConverter(Converter):
    247     def convert(self, src):
    248         data = super(FromFreeBSDToRTEMSSourceConverter, self).convert(src)
     335    def source_filter(self, data):
    249336        data = fixLocalIncludes(data)
    250337        data = fixIncludes(data)
     
    252339        return data
    253340
     341    def convert(self, src, dst):
     342        sconverter = super(FromFreeBSDToRTEMSSourceConverter, self)
     343        sconverter.convert(src, dst, source_filter = self.source_filter)
     344
    254345class FromFreeBSDToRTEMSUserSpaceSourceConverter(Converter):
    255     def convert(self, src):
    256         data = super(FromFreeBSDToRTEMSUserSpaceSourceConverter, self).convert(src)
     346    def source_filter(self, data):
    257347        data = fixIncludes(data)
    258348        data = '#include <machine/rtems-bsd-user-space.h>\n\n' + data
    259349        return data
    260350
     351    def convert(self, src, dst):
     352        sconverter = super(FromFreeBSDToRTEMSUserSpaceSourceConverter, self)
     353        sconverter.convert(src, dst, source_filter = self.source_filter)
     354
    261355class FromRTEMSToFreeBSDHeaderConverter(Converter):
    262     def convert(self, src):
    263         data = super(FromRTEMSToFreeBSDHeaderConverter, self).convert(src)
     356    def source_filter(self, data):
    264357        data = revertFixLocalIncludes(data)
    265358        data = revertFixIncludes(data)
    266359        return data
    267360
     361    def convert(self, src, dst):
     362        sconverter = super(FromRTEMSToFreeBSDHeaderConverter, self)
     363        sconverter.convert(src, dst, has_source = False,  source_filter = self.source_filter)
     364
    268365class FromRTEMSToFreeBSDSourceConverter(Converter):
    269     def convert(self, src):
    270         data = super(FromRTEMSToFreeBSDSourceConverter, self).convert(src)
     366    def source_filter(self, data):
    271367        data = re.sub('#include <machine/rtems-bsd-kernel-space.h>\n\n', '', data)
    272368        data = re.sub('#include <machine/rtems-bsd-user-space.h>\n\n', '', data)
    273         data = revertFixLocalIncludes(data)
    274         data = revertFixIncludes(data)
    275369        return data
    276370
     371    def convert(self, src, dst):
     372        sconverter = super(FromRTEMSToFreeBSDSourceConverter, self)
     373        sconverter.convert(src, dst, has_source = False, source_filter = self.source_filter)
     374
     375#
     376# Compose a path based for the various parts of the source tree.
     377#
    277378class PathComposer(object):
    278379    def composeFreeBSDPath(self, path):
     
    280381
    281382    def composeRTEMSPath(self, path, prefix):
    282         path = prefix + path
    283         return path
     383        return os.path.join(prefix, path)
    284384
    285385class FreeBSDPathComposer(PathComposer):
    286386    def composeFreeBSDPath(self, path):
    287         return FreeBSD_DIR + '/' + path
     387        return os.path.join(FreeBSD_DIR, path)
    288388
    289389    def composeRTEMSPath(self, path, prefix):
    290         return prefix + 'freebsd/' + path
     390        return os.path.join(prefix, 'freebsd', path)
    291391
    292392class RTEMSPathComposer(PathComposer):
     
    295395
    296396    def composeRTEMSPath(self, path, prefix):
    297         path = prefix + 'rtemsbsd/' + path
    298         return path
     397        return os.path.join(prefix, 'rtemsbsd', path)
    299398
    300399class CPUDependentPathComposer(FreeBSDPathComposer):
     
    324423        return ''
    325424
     425#
     426# File - a file in the source we move backwards and forwards.
     427#
    326428class File(object):
    327429    def __init__(self, path, pathComposer,
    328                  fromFreeBSDToRTEMSConverter, fromRTEMSToFreeBSDConverter, buildSystemComposer):
     430                 forwardConverter, reverseConverter, buildSystemComposer):
     431        if verbose(verboseMoreDetail):
     432            print("FILE: %-50s F:%-45s R:%-45s" % \
     433                  (path,
     434                   forwardConverter.__class__.__name__,
     435                   reverseConverter.__class__.__name__))
    329436        self.path = path
    330437        self.pathComposer = pathComposer
    331         self.fromFreeBSDToRTEMSConverter = fromFreeBSDToRTEMSConverter
    332         self.fromRTEMSToFreeBSDConverter = fromRTEMSToFreeBSDConverter
     438        self.freebsdPath = self.pathComposer.composeFreeBSDPath(self.path)
     439        self.rtemsPath = self.pathComposer.composeRTEMSPath(self.path, RTEMS_DIR)
     440        self.forwardConverter = forwardConverter
     441        self.reverseConverter = reverseConverter
    333442        self.buildSystemComposer = buildSystemComposer
    334443
    335     def copy(self, dst, src, converter = None):
    336         import tempfile
    337         if converter is not None and converter.isConvertible():
    338             try:
    339                 if isDryRun == False:
    340                     os.makedirs(os.path.dirname(dst))
    341             except OSError:
    342                 pass
    343             data = converter.convert(src)
    344             try:
    345                 out = tempfile.NamedTemporaryFile(delete = False)
    346                 out.write(data)
    347                 out.close()
    348                 processIfDifferent(out.name, dst, src)
    349             finally:
    350                 try:
    351                     os.remove(out.name)
    352                 except:
    353                     pass
    354 
    355     def copyFromFreeBSDToRTEMS(self):
    356         src = self.pathComposer.composeFreeBSDPath(self.path)
    357         dst = self.pathComposer.composeRTEMSPath(self.path, RTEMS_DIR + '/')
    358         self.copy(dst, src, self.fromFreeBSDToRTEMSConverter)
    359 
    360     def copyFromRTEMSToFreeBSD(self):
    361         src = self.pathComposer.composeRTEMSPath(self.path, RTEMS_DIR + '/')
    362         dst = self.pathComposer.composeFreeBSDPath(self.path)
    363         self.copy(dst, src, self.fromRTEMSToFreeBSDConverter)
     444    def processSource(self, forward):
     445        if forward:
     446            if verbose(verboseDetail):
     447                print("process source: %s => %s" % (self.freebsdPath, self.rtemsPath))
     448            self.forwardConverter.convert(self.freebsdPath, self.rtemsPath)
     449        else:
     450            if verbose(verboseDetail):
     451                print("process source: %s => %s converter:%s" % \
     452                      (self.rtemsPath, self.freebsdPath, self.reverseConverter.__class__.__name__))
     453            self.reverseConverter.convert(self.rtemsPath, self.freebsdPath)
    364454
    365455    def getFragment(self):
    366456        return self.buildSystemComposer.compose(self.pathComposer.composeRTEMSPath(self.path, ''))
    367457
     458#
    368459# Module - logical group of related files we can perform actions on
     460#
    369461class Module:
    370462    def __init__(self, name):
     
    379471            self.cpuDependentSourceFiles[cpu] = []
    380472
    381     def copyFromFreeBSDToRTEMS(self):
     473    def processSource(self, direction):
     474        if verbose(verboseDetail):
     475            print("process module: %s" % (self.name))
    382476        for f in self.files:
    383             f.copyFromFreeBSDToRTEMS()
     477            f.processSource(direction)
    384478        for cpu, files in self.cpuDependentSourceFiles.items():
    385479            for f in files:
    386                 f.copyFromFreeBSDToRTEMS()
    387 
    388     def copyFromRTEMSToFreeBSD(self):
    389         for f in self.files:
    390             f.copyFromRTEMSToFreeBSD()
    391         for cpu, files in self.cpuDependentSourceFiles.items():
    392             for f in files:
    393                 f.copyFromRTEMSToFreeBSD()
     480                f.processSource(direction)
    394481
    395482    def addFiles(self, newFiles, buildSystemComposer = BuildSystemFragmentComposer()):
     
    479566        self.dependencies += [dep]
    480567
     568#
     569# Manager - a collection of modules.
     570#
    481571class ModuleManager:
    482572    def __init__(self):
     
    496586        self.modules[module.name] = module
    497587
    498     def copyFromFreeBSDToRTEMS(self):
     588    def processSource(self, direction):
     589        if verbose(verboseDetail):
     590            print("process modules:")
    499591        for m in sorted(self.modules):
    500             self.modules[m].copyFromFreeBSDToRTEMS()
    501 
    502     def copyFromRTEMSToFreeBSD(self):
    503         for m in sorted(self.modules):
    504             self.modules[m].copyFromRTEMSToFreeBSD()
     592            self.modules[m].processSource(direction)
  • freebsd-to-rtems.py

    r5b93cc8 rf1fcdba  
    8888    for o, a in opts:
    8989        if o in ("-v", "--verbose"):
    90             builder.isVerbose = True
     90            builder.verboseLevel += 1
    9191        elif o in ("-h", "--help", "-?"):
    9292            usage()
     
    111111parseArguments()
    112112
    113 print("Verbose:                     " + ("no", "yes")[builder.isVerbose])
    114 print("Dry Run:                     " + ("no", "yes")[builder.isDryRun])
    115 print("Diff Mode Enabled:           " + ("no", "yes")[builder.isDiffMode])
    116 print("Only Generate Build Scripts: " + ("no", "yes")[isOnlyBuildScripts])
    117 print("RTEMS Libbsd Directory:      " + builder.RTEMS_DIR)
    118 print("FreeBSD SVN Directory:       " + builder.FreeBSD_DIR)
    119 print("Direction:                   " + ("reverse", "forward")[isForward])
     113print("Verbose:                     %s (%d)" % (("no", "yes")[builder.verbose()],
     114                                                builder.verboseLevel))
     115print("Dry Run:                     %s" % (("no", "yes")[builder.isDryRun]))
     116print("Diff Mode Enabled:           %s" % (("no", "yes")[builder.isDiffMode]))
     117print("Only Generate Build Scripts: %s" % (("no", "yes")[isOnlyBuildScripts]))
     118print("RTEMS Libbsd Directory:      %s" % (builder.RTEMS_DIR))
     119print("FreeBSD SVN Directory:       %s" % (builder.FreeBSD_DIR))
     120print("Direction:                   %s" % (("reverse", "forward")[isForward]))
    120121
    121122# Check directory argument was set and exist
     
    148149try:
    149150    waf_gen = waf_generator.ModuleManager()
    150 
    151151    libbsd.sources(waf_gen)
    152 
    153     # Perform the actual file manipulation
    154     if isForward:
    155         if not isOnlyBuildScripts:
    156             waf_gen.copyFromFreeBSDToRTEMS()
    157         waf_gen.generate(libbsd.rtems_version())
    158     else:
    159         waf_gen.copyFromRTEMSToFreeBSD()
     152    if not isOnlyBuildScripts:
     153        waf_gen.processSource(isForward)
     154    waf_gen.generate(libbsd.rtems_version())
    160155    builder.changedFileSummary()
    161156except IOError as ioe:
    162     print('error: %s' % (ioe))
     157    print('error: %s' % (str(ioe)))
     158except builder.error as be:
     159    print('error: %s' % (be))
     160except KeyboardInterrupt:
     161    print('user abort')
  • waf_generator.py

    r5b93cc8 rf1fcdba  
    148148
    149149    def write(self):
    150         try:
    151             out = tempfile.NamedTemporaryFile(delete = False)
    152             try:
    153                 out.write(bytes(self.script, sys.stdin.encoding))
    154             except:
    155                 out.write(self.script)
    156             out.close()
    157             wscript = builder.RTEMS_DIR + '/libbsd_waf.py'
    158             builder.processIfDifferent(out.name, wscript, "libbsd_waf.py")
    159         finally:
    160             try:
    161                 os.remove(out.name)
    162             except:
    163                 pass
     150        name = os.path.join(builder.RTEMS_DIR, 'libbsd_waf.py')
     151        converter = builder.Converter()
     152        converter.convert(name, name, src_contents = self.script)
    164153
    165154    def setGenerators(self):
Note: See TracChangeset for help on using the changeset viewer.