source: rtems-libbsd/userspace-header-gen.py @ b6c64e1

5-freebsd-12
Last change on this file since b6c64e1 was b2e1123, checked in by Sebastian Huber <sebastian.huber@…>, on Aug 28, 2018 at 12:14:27 PM

userspace-header-gen.py: Sort file content

Sort the file content to make it less dependent on compiler code
generation variances.

  • Property mode set to 100755
File size: 22.1 KB
Line 
1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4# Copyright (c) 2016 embedded brains GmbH.  All rights reserved.
5#
6#  embedded brains GmbH
7#  Dornierstr. 4
8#  82178 Puchheim
9#  Germany
10#  <rtems@embedded-brains.de>
11#
12# Redistribution and use in source and binary forms, with or without
13# modification, are permitted provided that the following conditions
14# are met:
15# 1. Redistributions of source code must retain the above copyright
16#    notice, this list of conditions and the following disclaimer.
17# 2. Redistributions in binary form must reproduce the above copyright
18#    notice, this list of conditions and the following disclaimer in the
19#    documentation and/or other materials provided with the distribution.
20#
21# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
22# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
25# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31# SUCH DAMAGE.
32
33from __future__ import print_function, division
34import argparse
35import sys
36from elftools.elf.elffile import ELFFile
37import re
38import copy
39import os
40
41VERBOSE_SOME = 1
42VERBOSE_MORE = 2
43VERBOSE_MOST = 3
44
45class Error(Exception):
46    """Base class for exceptions in this module."""
47    pass
48
49
50class NoDwarfInfoError(Error):
51    """Exception raised in case there is no DWARF information."""
52
53    def __init__(self):
54        super(NoDwarfInfoError, self).__init__("Input file has no DWARF info.")
55
56
57class TypenameNotFoundError(Error):
58    """Exception raised in case a die is not found like expected."""
59    pass
60
61
62class AnonymousStructureError(Error):
63    """Exception raised in case a die is not found like expected."""
64    pass
65
66
67class VarnameNotFoundError(Error):
68    """Exception raised in case a die is not found like expected."""
69
70    def __init__(self):
71        super(VarnameNotFoundError, self).__init__("Couldn't find the variables name.")
72
73
74class HeaderGenCU:
75    """Process a single CU"""
76
77    def __init__(self, cu, progname, lineprog, err = sys.stderr, verbose = 0,
78                 filterre = re.compile('.*')):
79        self._rtems_port_names = []
80        self._rtems_port_names.append("_Linker_set_bsd_prog_%s_begin" % progname)
81        self._rtems_port_names.append("_Linker_set_bsd_prog_%s_end" % progname)
82        self._rtems_port_names.append("rtems_bsd_command_%s" % progname)
83
84        self._filter_special_vars = []
85        # Take some special care for some yacc variables. This matches the yyval
86        # and yylval. These two always make trouble in the generated headers.
87        # yyval is initialized by Yacc generated code so it's not
88        # necessary to move them into the copy back region. yylval is used only
89        # for transporting a value. It will be set when used.
90        self._filter_special_vars.append({
91            "re":re.compile('extern YYSTYPE .*val'),
92            "reason":"Lex / Yacc variable initialized by generated code",
93            "action":"no_section"
94            })
95        # Lex generates an external variable that shouldn't be extern. Move it
96        # to the current data header file.
97        self._filter_special_vars.append({
98            "re":re.compile('extern yy_size_t .*len'),
99            "reason":"Lex adds an extern to this variable that is not necessary.",
100            "action":"ignore_extern"
101            })
102
103        self._err = err
104        self._verbose = verbose
105        self._cu = cu
106        self._progname = progname
107        self._die_by_offset = {}
108        self._lineprogram = lineprog
109        self._filterre = filterre
110        self._namespace_prefix = "_bsd_%s_" % (self._progname)
111
112        self._fill_die_list()
113
114        if self._verbose >= VERBOSE_MOST:
115            print('DIE list: \n', self._die_by_offset)
116
117    def _fill_die_list(self, die = None):
118        if die is None:
119            die = self._cu.get_top_DIE()
120        # Use relative indices for the keys like they are used to reference
121        # inside one cu
122        offset = die.offset - self._cu.cu_offset
123        self._die_by_offset[offset] = die
124        for child in die.iter_children():
125            self._fill_die_list(child)
126
127    def _die_is_var(self, die):
128        return (die.tag == "DW_TAG_variable")
129
130    def _die_is_function(self, die):
131        return (die.tag == "DW_TAG_subprogram")
132
133    def _get_type(self, die, first_array = True):
134        """Get the type of a variable DIE.
135        Returns two strings: one prefix and one postfix for the variable name"""
136        typepre = ""
137        typepost = ""
138
139        if self._verbose >= VERBOSE_MOST:
140            self._err.write('Search type for DIE with offset=%d\n' % \
141                            (die.offset))
142
143        try:
144            typedie_offset = die.attributes["DW_AT_type"].value
145        except KeyError:
146            raise TypenameNotFoundError('Couldn\'t find the offset of the type DIE\n')
147
148        try:
149            typedie = self._die_by_offset[typedie_offset]
150        except KeyError:
151            raise TypenameNotFoundError('Couldn\'t find the DIE at offset %d\n' % \
152                                        (typedie_offset))
153
154        last = False
155        if (typedie.tag == "DW_TAG_const_type"):
156            typepre += "const "
157
158        elif (typedie.tag == "DW_TAG_array_type"):
159            for child in typedie.iter_children():
160                if child.tag == "DW_TAG_subrange_type":
161                    if first_array == True:
162                        arraysize = ""
163                        first_array = False
164                    else:
165                        try:
166                            upper_bound = child.attributes["DW_AT_upper_bound"].value
167                            arraysize = "%d" % (upper_bound + 1)
168                        except KeyError:
169                            arraysize = ""
170                    typepost += "[%s]" % arraysize
171
172        elif (typedie.tag == "DW_TAG_volatile_type"):
173            typepre += "volatile "
174
175        elif (typedie.tag == "DW_TAG_pointer_type"):
176            typepre += "*"
177
178        elif (typedie.tag == "DW_TAG_structure_type"):
179            typepre += "struct "
180
181        elif (typedie.tag == "DW_TAG_enumeration_type"):
182            typepre += "enum "
183
184        elif (typedie.tag == "DW_TAG_subroutine_type"):
185            typepre = "("
186            typepost = ")("
187            current_child = 0
188            for child in typedie.iter_children():
189                pre, post = self._get_type(child)
190                if (current_child > 0):
191                    typepost += ", "
192                typepost += pre + post
193                current_child += 1
194            if current_child == 0:
195                typepost += "void"
196            typepost += ")"
197            if not "DW_AT_type" in typedie.attributes.keys():
198                typepre = "void " + typepre
199                last = True
200
201        elif (typedie.tag == "DW_TAG_typedef") or \
202             (typedie.tag == "DW_TAG_base_type"):
203            # nothing to do here than prevent the error
204            pass
205
206        else:
207            raise TypenameNotFoundError('Unknown tag: %s\n' % (typedie.tag))
208
209        if (typedie.tag == "DW_TAG_typedef") or \
210           (typedie.tag == "DW_TAG_base_type") or \
211           (typedie.tag == "DW_TAG_structure_type") or \
212           (typedie.tag == "DW_TAG_enumeration_type"):
213            last = True
214            try:
215                typepre += "%s " % \
216                          typedie.attributes["DW_AT_name"].value.decode('ascii')
217            except KeyError:
218                if typedie.has_children:
219                    message = 'Found an anonymous structure'
220                    raise AnonymousStructureError(message)
221                else:
222                    message = 'Couldn\'t get type name from DIE'
223                    raise TypenameNotFoundError(message)
224
225        if last == False:
226            addpre, addpost = self._get_type(typedie, first_array)
227            typepre = addpre + typepre
228            typepost = typepost + addpost
229
230        if self._verbose >= VERBOSE_MOST:
231            self._err.write('Add prefix="%s", postfix="%s" for DIE with offset=%d\n' % \
232                            (typepre, typepost, die.offset))
233
234        return typepre, typepost
235
236    def generate_header(self, data_out_filename, glob_data_out, namesp_out):
237        """Find all top level (global) variables in the ELF file and generate
238        output that can be written in a header.
239        """
240
241        top_die = self._cu.get_top_DIE()
242        try:
243            filename = top_die.attributes["DW_AT_name"].value.decode('ascii')
244        except KeyError:
245            filename = top_die.get_full_path()
246            self._err.write("WARNING: getting the filename failed. Use fallback.")
247
248        basename = os.path.basename(filename)
249        modulename = os.path.splitext(basename)[0]
250        my_data_out_filename = data_out_filename.replace("#MODULE#", modulename)
251        my_data_out = open(my_data_out_filename, "w")
252
253        glob_data_out.write("/* %s */\n" % (basename))
254
255        namesp_out.write("/* %s */\n" % (basename))
256
257        my_data_out.write("/* generated by userspace-header-gen.py */\n")
258        my_data_out.write("#include <rtems/linkersets.h>\n")
259        my_data_out.write('#include "%s"\n' % (glob_data_out.name))
260        my_data_out.write("/* %s */\n" % (basename))
261
262        self._process_die(top_die, my_data_out, glob_data_out, namesp_out)
263
264    def _is_constant(self, die):
265        is_constant = False
266        try:
267            type_offset = die.attributes["DW_AT_type"].value
268            typedie = self._die_by_offset[type_offset]
269        except KeyError:
270            self._err.write("WARNING: Could not find out whether DIE %d is const.\n" % \
271                            die.offset)
272            pass
273        else:
274            if typedie.tag == "DW_TAG_const_type":
275                is_constant = True
276        return is_constant
277
278    def _write_list_to_file(self, l, f):
279        if l:
280            l.sort()
281            f.write('\n'.join(l) + '\n')
282
283    def _process_die(self, die, data_out, glob_data_out, namesp_out):
284        data_out_items = []
285        glob_data_out_items = []
286        namesp_out_items = []
287
288        for child in die.iter_children():
289            specdie = child
290            # get the name of the DIE
291            try:
292                varname = child.attributes["DW_AT_name"].value.decode('ascii')
293            except KeyError:
294                # this might is an external variable with a specification
295                # located elsewhere
296                try:
297                    specification = child.attributes["DW_AT_specification"]\
298                                         .value
299                    specdie = self._die_by_offset[specification]
300                    varname = specdie.attributes["DW_AT_name"].value\
301                                     .decode('ascii')
302                except KeyError:
303                    varname = None
304
305            # filter all none variable or function DIEs
306            is_function = False
307            if self._die_is_var(child):
308                if self._verbose >= VERBOSE_MORE:
309                    self._err.write('Process variable DIE: tag=%s, name=%s\n' % \
310                                    (child.tag, varname))
311            elif self._die_is_function(child):
312                if self._verbose >= VERBOSE_MORE:
313                    self._err.write('Process function DIE: tag=%s, name=%s\n' % \
314                                    (child.tag, varname))
315                if varname is None:
316                    if self._verbose >= VERBOSE_MORE:
317                        self._err.write('Skip function with no name.\n')
318                    continue
319                is_function = True
320            else:
321                if self._verbose >= VERBOSE_MORE:
322                    self._err.write('DIE is no variable or function: tag=%s, name=%s\n' % \
323                                    (child.tag, varname))
324                    # FIXME: Check if this die has children and if one of the
325                    # children is a function static variable
326                continue
327
328            # filter some special names that are used for porting
329            if varname in self._rtems_port_names:
330                self._err.write('Skip %s. It is a special object for porting.\n' % \
331                                (varname))
332                continue
333
334            # check if it is an external variable
335            is_extern = False
336            try:
337                is_extern = (specdie.attributes["DW_AT_external"].value != 0)
338            except KeyError:
339                # if the key is not there it is not extern
340                is_extern = False
341
342            # check if it is an declaration
343            is_decl = False
344            try:
345                is_decl = (specdie.attributes["DW_AT_declaration"].value != 0)
346            except KeyError:
347                # if the key is not there it is not an declaration
348                is_decl = False
349
350            # filter declaration only lines (we only want the definitions)
351            if is_decl and specdie == child:
352                if self._verbose >= VERBOSE_MORE:
353                    self._err.write('Skip extern variable "%s" because it is only a declaration.\n' % \
354                    (varname))
355                continue
356
357            # filter constants
358            if (not is_function) and self._is_constant(specdie):
359                if self._verbose >= VERBOSE_SOME:
360                    self._err.write('Skip const variable "%s" because it is a const.\n' % (varname))
361                continue
362
363            # Check if we haven't found a name earlier
364            if varname is None:
365                raise VarnameNotFoundError
366
367            # Fixup name (necessary if the script runs a second time)
368            varname = varname.replace(self._namespace_prefix, "")
369
370            # get file and line
371            try:
372                decl_file_idx = child.attributes["DW_AT_decl_file"].value - 1
373                decl_file = self._lineprogram['file_entry'][decl_file_idx].name.decode('ascii')
374            except KeyError:
375                decl_file = "<unknown>"
376            try:
377                decl_line = child.attributes["DW_AT_decl_line"].value
378            except KeyError:
379                decl_line = "<unknown>"
380            var_decl = "%s:%s" % (decl_file, decl_line)
381
382            if self._filterre.match(decl_file) is None:
383                if self._verbose >= VERBOSE_SOME:
384                    self._err.write('Skip variable "%s" because it\'s declaration file (%s) doesn\'t match the filter\n' % \
385                                    (varname, var_decl))
386                continue
387
388            # get type for the variable
389            if not is_function:
390                try:
391                    typepre, typepost = self._get_type(specdie)
392                except TypenameNotFoundError:
393                    self._err.write('Couldn\'t find type for "%s" at %s\n' %
394                                    (varname, var_decl))
395                    raise
396                except AnonymousStructureError:
397                    self._err.write('ERROR: anonymous structure "%s" at %s\n' % \
398                                    (varname, var_decl))
399                    raise
400                var_with_type = "%s%s%s" % (typepre, varname, typepost)
401
402                # check if it is a static or a extern
403                if not is_extern:
404                    var_with_type = "static " + var_with_type
405                    out_items = data_out_items
406                else:
407                    self._err.write('WARNING: variable is not static: "%s" at %s\n' % \
408                                    (var_with_type, var_decl))
409                    var_with_type = "extern " + var_with_type
410                    out_items = glob_data_out_items
411
412                for flt in self._filter_special_vars:
413                    if flt["re"].match(var_with_type) is not None:
414                        if flt["action"] == "no_section":
415                            self._err.write('Don\'t put "%s" into section. Reason: %s.\n' % \
416                                            (var_with_type, flt["reason"]))
417                            out_items = None
418                        if flt["action"] == "ignore_extern":
419                            self._err.write('Ignore extern of variable "%s". Reason: %s.\n' % \
420                                            (var_with_type, flt["reason"]))
421                            out_items = data_out_items
422
423            # write output
424            if self._verbose >= VERBOSE_SOME:
425                if not is_function:
426                    self._err.write('Found a variable "%s" at %s (DIE offset %s); extern: %r\n' % \
427                                    (var_with_type, var_decl, child.offset, is_extern))
428                else:
429                    self._err.write('Found a function "%s" at %s (DIE offset %s); extern: %r\n' % \
430                                    (varname, var_decl, child.offset, is_extern))
431            if (not is_function) and (out_items is not None):
432                out_items.append("RTEMS_LINKER_RWSET_CONTENT(bsd_prog_%s, %s);" % \
433                        (self._progname, var_with_type))
434            if is_extern:
435                namesp_out_items.append("#define %s %s%s" % \
436                                 (varname, self._namespace_prefix, varname))
437
438        self._write_list_to_file(data_out_items, data_out)
439        self._write_list_to_file(glob_data_out_items, glob_data_out)
440        self._write_list_to_file(namesp_out_items, namesp_out)
441
442
443class UserspaceHeaderGen:
444    def __init__(self, objfiles, progname, err = sys.stderr, verbose = 0,
445                 filterre = re.compile(".*")):
446        self._err = err
447        self._verbose = verbose
448        self._objfiles = objfiles
449        self._progname = progname
450        self._filterre = filterre
451
452    def generate_header(self, data_out_filename, glob_data_out, namesp_out):
453        """Find all top level (global) variables in the ELF file and generate
454        a header.
455        """
456        glob_data_out.write("/* generated by userspace-header-gen.py */\n")
457        glob_data_out.write("#include <rtems/linkersets.h>\n")
458
459        namesp_out.write("/* generated by userspace-header-gen.py */\n")
460
461        for objfile in self._objfiles:
462            elffile = ELFFile(objfile)
463            if not elffile.has_dwarf_info():
464                raise NoDwarfInfoError()
465
466            # Don't relocate DWARF sections. This is not necessary for us but
467            # makes problems on ARM with current pyelftools (version 0.24)
468            dwarfinfo = elffile.get_dwarf_info(relocate_dwarf_sections=False)
469
470            for cu in dwarfinfo.iter_CUs():
471                if self._verbose >= VERBOSE_SOME:
472                    self._err.write('Found a CU at offset %s, length %s\n' % \
473                                    (cu.cu_offset, cu['unit_length']))
474
475                lineprog = dwarfinfo.line_program_for_CU(cu)
476                headergen = HeaderGenCU(cu, self._progname, lineprog, self._err,
477                                        self._verbose, self._filterre);
478                headergen.generate_header(data_out_filename, glob_data_out,
479                                          namesp_out);
480
481
482if __name__ == '__main__':
483    default_filter = '.*'
484    default_dataout = 'rtems-bsd-#PROGNAME#-#MODULE#-data.h'
485    default_globdataout = 'rtems-bsd-#PROGNAME#-data.h'
486    default_namespaceout = 'rtems-bsd-#PROGNAME#-namespace.h'
487    parser = argparse.ArgumentParser(
488        description=(
489            "Generate header files for porting FreeBSD user space tools to RTEMS."
490            "Takes an object file as input."
491        ))
492    parser.add_argument(
493        "objfile",
494        help="Text arguments. One or more can be appended to the call.",
495        type=argparse.FileType("rb"),
496        nargs='+'
497    )
498    parser.add_argument(
499        "-f", "--filter",
500        help="Only process variables that are defined in files with a name " \
501             "matching the given regular expression. " \
502             "Default: '%s'" % default_filter,
503        dest="filter_string",
504        default=default_filter
505    )
506    parser.add_argument(
507        "-p", "--progname",
508        help="Name of the program. Default: MYPROG",
509        default="MYPROG"
510    )
511    parser.add_argument(
512        "-d", "--dataout",
513        help="Name of the output files where the section attributes will be " \
514             "added. '#PROGNAME#' will be replaced by the program name " \
515             "(set by parameter -p). '#MODULE#' will be replaced by the "
516             "current c modules base name. " \
517             "Default: '%s'" % (default_dataout),
518        default=default_dataout,
519        nargs="?"
520    )
521    parser.add_argument(
522        "-g", "--globdataout",
523        help="Name of the output files where the section attributes for " \
524             "global variables will be added. " \
525             "Default: '%s'" % (default_globdataout),
526        default=default_globdataout,
527        nargs="?"
528    )
529    parser.add_argument(
530        "-n", "--namespaceout",
531        help="Name of the output file where namespace definitions will be " \
532             "added. Default: '%s'" % (default_namespaceout),
533        default=default_namespaceout,
534        nargs="?"
535    )
536    parser.add_argument(
537        "-v", "--verbose",
538        help="Be more verbose. Can be used multiple times.",
539        default=0,
540        action="count"
541    )
542    args = parser.parse_args()
543
544    filterre = re.compile(args.filter_string)
545
546    globdataoutfilename = args.globdataout.replace("#PROGNAME#", args.progname)
547    globdataoutfile = open(globdataoutfilename, 'w')
548
549    namespaceoutfilename = args.namespaceout.replace("#PROGNAME#", args.progname)
550    namespaceoutfile = open(namespaceoutfilename, 'w')
551
552    dataoutfilename = args.dataout.replace("#PROGNAME#", args.progname)
553
554    uhg = UserspaceHeaderGen(objfiles = args.objfile,
555                             verbose = args.verbose,
556                             progname = args.progname,
557                             filterre = filterre)
558    uhg.generate_header(dataoutfilename, globdataoutfile, namespaceoutfile)
559
560# vim: set ts=4 sw=4 et:
Note: See TracBrowser for help on using the repository browser.