source: rtems-libbsd/userspace-header-gen.py @ c7eec93

55-freebsd-126-freebsd-12
Last change on this file since c7eec93 was 879eaa7, checked in by Christian Mauderer <christian.mauderer@…>, on 06/13/18 at 13:04:47

userspace-header-gen: Special case for Yacc / Lex.

Some variables generated by Yacc / Lex need a special treatment. That
had been done by manual editing of the generated headers in the past.
This patch integrate these changes into the generator.

  • Property mode set to 100755
File size: 21.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright (c) 2016 embedded brains GmbH.  All rights reserved.
5#
6#  embedded brains GmbH
7#  Dornierstr. 4
8#  82178 Puchheim
9#  Germany
10#  <rtems@embedded-brains.de>
11#
12# Redistribution and use in source and binary forms, with or without
13# modification, are permitted provided that the following conditions
14# are met:
15# 1. Redistributions of source code must retain the above copyright
16#    notice, this list of conditions and the following disclaimer.
17# 2. Redistributions in binary form must reproduce the above copyright
18#    notice, this list of conditions and the following disclaimer in the
19#    documentation and/or other materials provided with the distribution.
20#
21# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
22# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
25# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31# SUCH DAMAGE.
32
33from __future__ import print_function, division
34import argparse
35import sys
36from elftools.elf.elffile import ELFFile
37import re
38import copy
39import os
40
41VERBOSE_SOME = 1
42VERBOSE_MORE = 2
43VERBOSE_MOST = 3
44
45class Error(Exception):
46    """Base class for exceptions in this module."""
47    pass
48
49
50class NoDwarfInfoError(Error):
51    """Exception raised in case there is no DWARF information."""
52
53    def __init__(self):
54        super(NoDwarfInfoError, self).__init__("Input file has no DWARF info.")
55
56
57class TypenameNotFoundError(Error):
58    """Exception raised in case a die is not found like expected."""
59    pass
60
61
62class AnonymousStructureError(Error):
63    """Exception raised in case a die is not found like expected."""
64    pass
65
66
67class VarnameNotFoundError(Error):
68    """Exception raised in case a die is not found like expected."""
69
70    def __init__(self):
71        super(VarnameNotFoundError, self).__init__("Couldn't find the variables name.")
72
73
74class HeaderGenCU:
75    """Process a single CU"""
76
77    def __init__(self, cu, progname, lineprog, err = sys.stderr, verbose = 0,
78                 filterre = re.compile('.*')):
79        self._rtems_port_names = []
80        self._rtems_port_names.append("_Linker_set_bsd_prog_%s_begin" % progname)
81        self._rtems_port_names.append("_Linker_set_bsd_prog_%s_end" % progname)
82        self._rtems_port_names.append("rtems_bsd_command_%s" % progname)
83
84        self._filter_special_vars = []
85        # Take some special care for some yacc variables. This matches the yyval
86        # and yylval. These two always make trouble in the generated headers.
87        # yyval is initialized by Yacc generated code so it's not
88        # necessary to move them into the copy back region. yylval is used only
89        # for transporting a value. It will be set when used.
90        self._filter_special_vars.append({
91            "re":re.compile('extern YYSTYPE .*val'),
92            "reason":"Lex / Yacc variable initialized by generated code",
93            "action":"no_section"
94            })
95        # Lex generates an external variable that shouldn't be extern. Move it
96        # to the current data header file.
97        self._filter_special_vars.append({
98            "re":re.compile('extern yy_size_t .*len'),
99            "reason":"Lex adds an extern to this variable that is not necessary.",
100            "action":"ignore_extern"
101            })
102
103        self._err = err
104        self._verbose = verbose
105        self._cu = cu
106        self._progname = progname
107        self._die_by_offset = {}
108        self._lineprogram = lineprog
109        self._filterre = filterre
110        self._namespace_prefix = "_bsd_%s_" % (self._progname)
111
112        self._fill_die_list()
113
114        if self._verbose >= VERBOSE_MOST:
115            print('DIE list: \n', self._die_by_offset)
116
117    def _fill_die_list(self, die = None):
118        if die is None:
119            die = self._cu.get_top_DIE()
120        # Use relative indices for the keys like they are used to reference
121        # inside one cu
122        offset = die.offset - self._cu.cu_offset
123        self._die_by_offset[offset] = die
124        for child in die.iter_children():
125            self._fill_die_list(child)
126
127    def _die_is_var(self, die):
128        return (die.tag == "DW_TAG_variable")
129
130    def _die_is_function(self, die):
131        return (die.tag == "DW_TAG_subprogram")
132
133    def _get_type(self, die, first_array = True):
134        """Get the type of a variable DIE.
135        Returns two strings: one prefix and one postfix for the variable name"""
136        typepre = ""
137        typepost = ""
138
139        if self._verbose >= VERBOSE_MOST:
140            self._err.write('Search type for DIE with offset=%d\n' % \
141                            (die.offset))
142
143        try:
144            typedie_offset = die.attributes["DW_AT_type"].value
145        except KeyError:
146            raise TypenameNotFoundError('Couldn\'t find the offset of the type DIE\n')
147
148        try:
149            typedie = self._die_by_offset[typedie_offset]
150        except KeyError:
151            raise TypenameNotFoundError('Couldn\'t find the DIE at offset %d\n' % \
152                                        (typedie_offset))
153
154        last = False
155        if (typedie.tag == "DW_TAG_const_type"):
156            typepre += "const "
157
158        elif (typedie.tag == "DW_TAG_array_type"):
159            for child in typedie.iter_children():
160                if child.tag == "DW_TAG_subrange_type":
161                    if first_array == True:
162                        arraysize = ""
163                        first_array = False
164                    else:
165                        try:
166                            upper_bound = child.attributes["DW_AT_upper_bound"].value
167                            arraysize = "%d" % (upper_bound + 1)
168                        except KeyError:
169                            arraysize = ""
170                    typepost += "[%s]" % arraysize
171
172        elif (typedie.tag == "DW_TAG_volatile_type"):
173            typepre += "volatile "
174
175        elif (typedie.tag == "DW_TAG_pointer_type"):
176            typepre += "*"
177
178        elif (typedie.tag == "DW_TAG_structure_type"):
179            typepre += "struct "
180
181        elif (typedie.tag == "DW_TAG_enumeration_type"):
182            typepre += "enum "
183
184        elif (typedie.tag == "DW_TAG_subroutine_type"):
185            typepre = "("
186            typepost = ")("
187            current_child = 0
188            for child in typedie.iter_children():
189                pre, post = self._get_type(child)
190                if (current_child > 0):
191                    typepost += ", "
192                typepost += pre + post
193                current_child += 1
194            if current_child == 0:
195                typepost += "void"
196            typepost += ")"
197            if not "DW_AT_type" in typedie.attributes.keys():
198                typepre = "void " + typepre
199                last = True
200
201        elif (typedie.tag == "DW_TAG_typedef") or \
202             (typedie.tag == "DW_TAG_base_type"):
203            # nothing to do here than prevent the error
204            pass
205
206        else:
207            raise TypenameNotFoundError('Unknown tag: %s\n' % (typedie.tag))
208
209        if (typedie.tag == "DW_TAG_typedef") or \
210           (typedie.tag == "DW_TAG_base_type") or \
211           (typedie.tag == "DW_TAG_structure_type") or \
212           (typedie.tag == "DW_TAG_enumeration_type"):
213            last = True
214            try:
215                typepre += "%s " % \
216                          typedie.attributes["DW_AT_name"].value.decode('ascii')
217            except KeyError:
218                if typedie.has_children:
219                    message = 'Found an anonymous structure'
220                    raise AnonymousStructureError(message)
221                else:
222                    message = 'Couldn\'t get type name from DIE'
223                    raise TypenameNotFoundError(message)
224
225        if last == False:
226            addpre, addpost = self._get_type(typedie, first_array)
227            typepre = addpre + typepre
228            typepost = typepost + addpost
229
230        if self._verbose >= VERBOSE_MOST:
231            self._err.write('Add prefix="%s", postfix="%s" for DIE with offset=%d\n' % \
232                            (typepre, typepost, die.offset))
233
234        return typepre, typepost
235
236    def generate_header(self, data_out_filename, glob_data_out, namesp_out):
237        """Find all top level (global) variables in the ELF file and generate
238        output that can be written in a header.
239        """
240
241        top_die = self._cu.get_top_DIE()
242        try:
243            filename = top_die.attributes["DW_AT_name"].value.decode('ascii')
244        except KeyError:
245            filename = top_die.get_full_path()
246            self._err.write("WARNING: getting the filename failed. Use fallback.")
247
248        basename = os.path.basename(filename)
249        modulename = os.path.splitext(basename)[0]
250        my_data_out_filename = data_out_filename.replace("#MODULE#", modulename)
251        my_data_out = open(my_data_out_filename, "w")
252
253        glob_data_out.write("/* %s */\n" % (basename))
254
255        namesp_out.write("/* %s */\n" % (basename))
256
257        my_data_out.write("/* generated by userspace-header-gen.py */\n")
258        my_data_out.write("#include <rtems/linkersets.h>\n")
259        my_data_out.write('#include "%s"\n' % (glob_data_out.name))
260        my_data_out.write("/* %s */\n" % (basename))
261
262        self._process_die(top_die, my_data_out, glob_data_out, namesp_out)
263
264    def _is_constant(self, die):
265        is_constant = False
266        try:
267            type_offset = die.attributes["DW_AT_type"].value
268            typedie = self._die_by_offset[type_offset]
269        except KeyError:
270            self._err.write("WARNING: Could not find out whether DIE %d is const.\n" % \
271                            die.offset)
272            pass
273        else:
274            if typedie.tag == "DW_TAG_const_type":
275                is_constant = True
276        return is_constant
277
278    def _process_die(self, die, data_out, glob_data_out, namesp_out):
279        for child in die.iter_children():
280            specdie = child
281            # get the name of the DIE
282            try:
283                varname = child.attributes["DW_AT_name"].value.decode('ascii')
284            except KeyError:
285                # this might is an external variable with a specification
286                # located elsewhere
287                try:
288                    specification = child.attributes["DW_AT_specification"]\
289                                         .value
290                    specdie = self._die_by_offset[specification]
291                    varname = specdie.attributes["DW_AT_name"].value\
292                                     .decode('ascii')
293                except KeyError:
294                    varname = None
295
296            # filter all none variable or function DIEs
297            is_function = False
298            if self._die_is_var(child):
299                if self._verbose >= VERBOSE_MORE:
300                    self._err.write('Process variable DIE: tag=%s, name=%s\n' % \
301                                    (child.tag, varname))
302            elif self._die_is_function(child):
303                if self._verbose >= VERBOSE_MORE:
304                    self._err.write('Process function DIE: tag=%s, name=%s\n' % \
305                                    (child.tag, varname))
306                if varname is None:
307                    if self._verbose >= VERBOSE_MORE:
308                        self._err.write('Skip function with no name.\n')
309                    continue
310                is_function = True
311            else:
312                if self._verbose >= VERBOSE_MORE:
313                    self._err.write('DIE is no variable or function: tag=%s, name=%s\n' % \
314                                    (child.tag, varname))
315                    # FIXME: Check if this die has children and if one of the
316                    # children is a function static variable
317                continue
318
319            # filter some special names that are used for porting
320            if varname in self._rtems_port_names:
321                self._err.write('Skip %s. It is a special object for porting.\n' % \
322                                (varname))
323                continue
324
325            # check if it is an external variable
326            is_extern = False
327            try:
328                is_extern = (specdie.attributes["DW_AT_external"].value != 0)
329            except KeyError:
330                # if the key is not there it is not extern
331                is_extern = False
332
333            # check if it is an declaration
334            is_decl = False
335            try:
336                is_decl = (specdie.attributes["DW_AT_declaration"].value != 0)
337            except KeyError:
338                # if the key is not there it is not an declaration
339                is_decl = False
340
341            # filter declaration only lines (we only want the definitions)
342            if is_decl and specdie == child:
343                if self._verbose >= VERBOSE_MORE:
344                    self._err.write('Skip extern variable "%s" because it is only a declaration.\n' % \
345                    (varname))
346                continue
347
348            # filter constants
349            if (not is_function) and self._is_constant(specdie):
350                if self._verbose >= VERBOSE_SOME:
351                    self._err.write('Skip const variable "%s" because it is a const.\n' % (varname))
352                continue
353
354            # Check if we haven't found a name earlier
355            if varname is None:
356                raise VarnameNotFoundError
357
358            # Fixup name (necessary if the script runs a second time)
359            varname = varname.replace(self._namespace_prefix, "")
360
361            # get file and line
362            try:
363                decl_file_idx = child.attributes["DW_AT_decl_file"].value - 1
364                decl_file = self._lineprogram['file_entry'][decl_file_idx].name.decode('ascii')
365            except KeyError:
366                decl_file = "<unknown>"
367            try:
368                decl_line = child.attributes["DW_AT_decl_line"].value
369            except KeyError:
370                decl_line = "<unknown>"
371            var_decl = "%s:%s" % (decl_file, decl_line)
372
373            if self._filterre.match(decl_file) is None:
374                if self._verbose >= VERBOSE_SOME:
375                    self._err.write('Skip variable "%s" because it\'s declaration file (%s) doesn\'t match the filter\n' % \
376                                    (varname, var_decl))
377                continue
378
379            # get type for the variable
380            if not is_function:
381                try:
382                    typepre, typepost = self._get_type(specdie)
383                except TypenameNotFoundError:
384                    self._err.write('Couldn\'t find type for "%s" at %s\n' %
385                                    (varname, var_decl))
386                    raise
387                except AnonymousStructureError:
388                    self._err.write('ERROR: anonymous structure "%s" at %s\n' % \
389                                    (varname, var_decl))
390                    raise
391                var_with_type = "%s%s%s" % (typepre, varname, typepost)
392
393                # check if it is a static or a extern
394                if not is_extern:
395                    var_with_type = "static " + var_with_type
396                    outfile = data_out
397                else:
398                    self._err.write('WARNING: variable is not static: "%s" at %s\n' % \
399                                    (var_with_type, var_decl))
400                    var_with_type = "extern " + var_with_type
401                    outfile = glob_data_out
402
403                for flt in self._filter_special_vars:
404                    if flt["re"].match(var_with_type) is not None:
405                        if flt["action"] == "no_section":
406                            self._err.write('Don\'t put "%s" into section. Reason: %s.\n' % \
407                                            (var_with_type, flt["reason"]))
408                            outfile = None
409                        if flt["action"] == "ignore_extern":
410                            self._err.write('Ignore extern of variable "%s". Reason: %s.\n' % \
411                                            (var_with_type, flt["reason"]))
412                            outfile = data_out
413
414            # write output
415            if self._verbose >= VERBOSE_SOME:
416                if not is_function:
417                    self._err.write('Found a variable "%s" at %s (DIE offset %s); extern: %r\n' % \
418                                    (var_with_type, var_decl, child.offset, is_extern))
419                else:
420                    self._err.write('Found a function "%s" at %s (DIE offset %s); extern: %r\n' % \
421                                    (varname, var_decl, child.offset, is_extern))
422            if (not is_function) and (outfile is not None):
423                outfile.write("RTEMS_LINKER_RWSET_CONTENT(bsd_prog_%s, %s);\n" % \
424                        (self._progname, var_with_type))
425            if is_extern:
426                namesp_out.write("#define %s %s%s\n" % \
427                                 (varname, self._namespace_prefix, varname))
428
429
430class UserspaceHeaderGen:
431    def __init__(self, objfiles, progname, err = sys.stderr, verbose = 0,
432                 filterre = re.compile(".*")):
433        self._err = err
434        self._verbose = verbose
435        self._objfiles = objfiles
436        self._progname = progname
437        self._filterre = filterre
438
439    def generate_header(self, data_out_filename, glob_data_out, namesp_out):
440        """Find all top level (global) variables in the ELF file and generate
441        a header.
442        """
443        glob_data_out.write("/* generated by userspace-header-gen.py */\n")
444        glob_data_out.write("#include <rtems/linkersets.h>\n")
445
446        namesp_out.write("/* generated by userspace-header-gen.py */\n")
447
448        for objfile in self._objfiles:
449            elffile = ELFFile(objfile)
450            if not elffile.has_dwarf_info():
451                raise NoDwarfInfoError()
452
453            # Don't relocate DWARF sections. This is not necessary for us but
454            # makes problems on ARM with current pyelftools (version 0.24)
455            dwarfinfo = elffile.get_dwarf_info(relocate_dwarf_sections=False)
456
457            for cu in dwarfinfo.iter_CUs():
458                if self._verbose >= VERBOSE_SOME:
459                    self._err.write('Found a CU at offset %s, length %s\n' % \
460                                    (cu.cu_offset, cu['unit_length']))
461
462                lineprog = dwarfinfo.line_program_for_CU(cu)
463                headergen = HeaderGenCU(cu, self._progname, lineprog, self._err,
464                                        self._verbose, self._filterre);
465                headergen.generate_header(data_out_filename, glob_data_out,
466                                          namesp_out);
467
468
469if __name__ == '__main__':
470    default_filter = '.*'
471    default_dataout = 'rtems-bsd-#PROGNAME#-#MODULE#-data.h'
472    default_globdataout = 'rtems-bsd-#PROGNAME#-data.h'
473    default_namespaceout = 'rtems-bsd-#PROGNAME#-namespace.h'
474    parser = argparse.ArgumentParser(
475        description=(
476            "Generate header files for porting FreeBSD user space tools to RTEMS."
477            "Takes an object file as input."
478        ))
479    parser.add_argument(
480        "objfile",
481        help="Text arguments. One or more can be appended to the call.",
482        type=argparse.FileType("rb"),
483        nargs='+'
484    )
485    parser.add_argument(
486        "-f", "--filter",
487        help="Only process variables that are defined in files with a name " \
488             "matching the given regular expression. " \
489             "Default: '%s'" % default_filter,
490        dest="filter_string",
491        default=default_filter
492    )
493    parser.add_argument(
494        "-p", "--progname",
495        help="Name of the program. Default: MYPROG",
496        default="MYPROG"
497    )
498    parser.add_argument(
499        "-d", "--dataout",
500        help="Name of the output files where the section attributes will be " \
501             "added. '#PROGNAME#' will be replaced by the program name " \
502             "(set by parameter -p). '#MODULE#' will be replaced by the "
503             "current c modules base name. " \
504             "Default: '%s'" % (default_dataout),
505        default=default_dataout,
506        nargs="?"
507    )
508    parser.add_argument(
509        "-g", "--globdataout",
510        help="Name of the output files where the section attributes for " \
511             "global variables will be added. " \
512             "Default: '%s'" % (default_globdataout),
513        default=default_globdataout,
514        nargs="?"
515    )
516    parser.add_argument(
517        "-n", "--namespaceout",
518        help="Name of the output file where namespace definitions will be " \
519             "added. Default: '%s'" % (default_namespaceout),
520        default=default_namespaceout,
521        nargs="?"
522    )
523    parser.add_argument(
524        "-v", "--verbose",
525        help="Be more verbose. Can be used multiple times.",
526        default=0,
527        action="count"
528    )
529    args = parser.parse_args()
530
531    filterre = re.compile(args.filter_string)
532
533    globdataoutfilename = args.globdataout.replace("#PROGNAME#", args.progname)
534    globdataoutfile = open(globdataoutfilename, 'w')
535
536    namespaceoutfilename = args.namespaceout.replace("#PROGNAME#", args.progname)
537    namespaceoutfile = open(namespaceoutfilename, 'w')
538
539    dataoutfilename = args.dataout.replace("#PROGNAME#", args.progname)
540
541    uhg = UserspaceHeaderGen(objfiles = args.objfile,
542                             verbose = args.verbose,
543                             progname = args.progname,
544                             filterre = filterre)
545    uhg.generate_header(dataoutfilename, globdataoutfile, namespaceoutfile)
546
547# vim: set ts=4 sw=4 et:
Note: See TracBrowser for help on using the repository browser.