source: rtems-docs/common/waf.py @ 07f151f

5
Last change on this file since 07f151f was 07f151f, checked in by Chris Johns <chrisj@…>, on 11/10/22 at 04:32:44

waf: Backport from main build fixes

Closes #4752
  • Property mode set to 100644
File size: 21.7 KB
Line 
1#
2# RTEMS Documentation Project
3#
4# Waf build support.
5#
6
7
8from __future__ import print_function
9
10import os
11import re
12import sys
13
14from waflib.Build import BuildContext
15
16import latex
17import conf
18
19sphinx_min_version = (1, 3)
20
21def version_cmdline(ctx):
22    return '-Drelease="%s" -Dversion="%s" -Drtems_major="%s" ' \
23           '-Drtems_minor="%s" -D rtems_revision="%s"' \
24        % (ctx.env.RELEASE, ctx.env.VERSION, ctx.env.RTEMS_MAJOR,
25           ctx.env.RTEMS_MINOR, ctx.env.RTEMS_REVISION)
26
27def sphinx_cmdline(ctx, build_type, conf_dir, doctrees,
28                   source_dir, output_dir, configs = []):
29    cfgs = ''
30    for c in configs:
31        cfgs += ' -D %s=%s' % (c, configs[c])
32    rule = "${BIN_SPHINX_BUILD} %s -b %s -c %s %s -d %s %s %s %s ${SRC}" % \
33           (sphinx_options(ctx), build_type, conf_dir, version_cmdline(ctx),
34            doctrees, cfgs, source_dir, output_dir)
35    return rule
36
37def cmd_spell(ctx):
38    from waflib import Options
39    from sys import argv
40    from subprocess import call
41
42    Options.commands = None # stop warnings about knowing commands.
43
44    if not ctx.env.BIN_ASPELL:
45        ctx.fatal("'aspell' is required please install and re-run configure.")
46
47    if len(argv) < 3:
48        ctx.fatal("Please supply at least one file name")
49
50    files = argv[2:]
51
52    path = ctx.path.parent.abspath()
53
54    # XXX: add error checking eg check if file exists.
55    for file in files:
56        cmd = ctx.env.BIN_ASPELL + \
57              ["-c",
58               "--personal=%s/common/spell/dict/rtems" % path,
59               "--extra-dicts=%s/common/spell/en_GB-ise-w_accents.multi" % path,
60               file]
61        print("running:", cmd)
62        call(cmd)
63
64def cmd_linkcheck(ctx):
65    conf_dir = ctx.path.get_src()
66    source_dir = ctx.path.get_src()
67    buildtype = 'linkcheck'
68    build_dir, output_node, output_dir, doctrees = build_dir_setup(ctx, buildtype)
69    rule = sphinx_cmdline(ctx, buildtype, conf_dir, doctrees, source_dir, output_dir)
70    ctx(
71        rule   = rule,
72        cwd    = ctx.path.abspath(),
73        source = ctx.path.ant_glob('**/*.rst'),
74        target = "linkcheck/output.txt"
75    )
76
77class spell(BuildContext):
78    __doc__ = "Check spelling.  Supply a list of files or a glob (*.rst)"
79    cmd = 'spell'
80    fun = 'cmd_spell'
81
82class linkcheck(BuildContext):
83    __doc__ = "Check all external URL references."
84    cmd = 'linkcheck'
85    fun = 'cmd_linkcheck'
86
87def check_sphinx_version(ctx, minver):
88    try:
89        import sphinx
90    except:
91        ctx.fatal('no sphinx support found; please install')
92    try:
93        # sphinx.version_info was introduced in sphinx ver 1.2
94        version = sphinx.version_info
95        # version looks like (1, 7, 0, 'final', 0))
96        ver = version[0:2]
97    except:
98        try:
99            # sphinx-build returns its version info in stderr
100            (out, err) = ctx.cmd_and_log(ctx.env.BIN_SPHINX_BUILD +
101                              ['--version'], output=Context.BOTH)
102            # err looks like 'sphinx-build 1.7.0\n'
103            version = err.split(" ")[-1:][0].strip()
104            ver = tuple(map(int, re.split('[\D]', version)))
105        except:
106            try:
107                # sphinx-build returns its version info in stdout
108                version = ctx.cmd_and_log(ctx.env.BIN_SPHINX_BUILD +
109                              ['--version']).split(" ")[-1:][0].strip()
110                try:
111                    ver = tuple(map(int, re.split('[\D]', version)))
112                except:
113                    ctx.fatal("Sphinx version cannot be checked or Sphinx is not installed")
114            except:
115                ctx.fatal("Sphinx version cannot be checked or Sphinx is not installed")
116    if ver < minver:
117        ctx.fatal("Sphinx version is too old: %s" % ".".join(map(str, ver)))
118    return ver
119
120def sphinx_options(ctx):
121    return ' '.join(ctx.env.SPHINX_OPTIONS)
122
123def is_top_build(ctx):
124    from_top = False
125    if ctx.env['BUILD_FROM_TOP'] and ctx.env['BUILD_FROM_TOP'] == 'yes':
126        from_top = True
127    return from_top
128
129def build_dir_setup(ctx, buildtype):
130    where = buildtype
131    if is_top_build(ctx):
132        where = os.path.join(ctx.path.name, where)
133    bnode = ctx.bldnode.find_node(where)
134    if bnode is None:
135        ctx.bldnode.make_node(where).mkdir()
136    build_dir = ctx.path.get_bld().relpath()
137    output_node = ctx.path.get_bld().make_node(buildtype)
138    output_dir = output_node.abspath()
139    doctrees = os.path.join(os.path.dirname(output_dir), 'doctrees', buildtype)
140    return build_dir, output_node, output_dir, doctrees
141
142def pdf_resources(ctx, buildtype):
143    packages_base = ctx.path.parent.find_dir('common/latex')
144    if packages_base is None:
145        ctx.fatal('Latex package directory not found')
146    base = packages_base.path_from(ctx.path)
147    fnode = ctx.path.get_bld().make_node(buildtype)
148    fnode.mkdir()
149    local_packages = latex.local_packages()
150    targets = []
151    if local_packages is not None:
152        srcs = [os.path.join(base, p) for p in local_packages]
153        targets += [fnode.make_node(p) for p in local_packages]
154        ctx(features = "subst",
155            is_copy  = True,
156            source   = srcs,
157            target   = targets)
158    targets += [fnode.make_node('rtemsextrafonts.sty')]
159    ctx(features = "subst",
160        is_copy  = True,
161        source   = os.path.join(base, ctx.env.RTEMSEXTRAFONTS),
162        target   = fnode.make_node('rtemsextrafonts.sty'))
163    return targets
164
165def html_resources(ctx, buildtype):
166    resources = []
167    for dir_name in ["_static", "_templates"]:
168        files = ctx.path.parent.find_node("common").ant_glob("%s/*" % dir_name)
169        fnode = ctx.path.get_bld().make_node(os.path.join(buildtype, dir_name))
170        targets = [fnode.make_node(x.name) for x in files]
171        resources += targets
172        fnode.mkdir() # dirs
173        ctx(features = "subst",
174            is_copy  = True,
175            source   = files,
176            target   = targets)
177        ctx.add_group()
178    return resources
179
180def check_sphinx_extension(ctx, extension):
181    def run_sphinx(bld):
182        rst_node = bld.srcnode.make_node('testbuild/contents.rst')
183        rst_node.parent.mkdir()
184        rst_node.write('.. COMMENT test sphinx' + os.linesep)
185        bib_node = bld.srcnode.make_node('testbuild/refs.bib')
186        bib_node.write(os.linesep)
187        conf_node = bld.srcnode.make_node('testbuild/conf.py')
188        conf_node.write(os.linesep.join(["master_doc='contents'",
189                                         "bibtex_bibfiles = ['refs.bib']"]))
190        bld(rule = bld.kw['rule'], source = rst_node)
191
192    ctx.start_msg("Checking for '%s'" % (extension))
193    try:
194        ctx.run_build(fragment = 'xx',
195                      rule = "${BIN_SPHINX_BUILD} -b html -D extensions=%s -c . . out" % (extension),
196                      build_fun = run_sphinx,
197                      env = ctx.env)
198    except ctx.errors.ConfigurationError:
199        ctx.end_msg('not found (see README.txt)', 'RED')
200        ctx.fatal('The configuration failed')
201    ctx.end_msg('found')
202
203def cmd_configure(ctx):
204    check_sphinx = not ctx.env.BIN_SPHINX_BUILD
205    if check_sphinx:
206        ctx.find_program("sphinx-build", var="BIN_SPHINX_BUILD", mandatory = True)
207        ctx.find_program("aspell", var = "BIN_ASPELL", mandatory = False)
208
209        ctx.start_msg("Checking if Sphinx is at least %s.%s" % sphinx_min_version)
210        ver = check_sphinx_version(ctx, sphinx_min_version)
211        ctx.end_msg("yes (%s)" % ".".join(map(str, ver)))
212
213        ctx.start_msg("Checking Sphinx Options ")
214        if 'SPHINX_OPTIONS' not in ctx.env:
215            ctx.env.append_value('SPHINX_OPTIONS', ctx.options.sphinx_options)
216            opts = sphinx_options(ctx)
217            if len(opts) == 0:
218                opts = 'none'
219            ctx.end_msg(opts)
220
221        ctx.start_msg("Checking Sphinx Nit-Pick mode ")
222        if ctx.options.sphinx_nit_pick:
223            opt = '-n'
224            msg = 'yes'
225        else:
226            opt = '-Q'
227            msg = 'no'
228        ctx.env.append_value('SPHINX_OPTIONS', opt)
229        ctx.end_msg(msg)
230
231        #
232        # Check extensions.
233        #
234        check_sphinx_extension(ctx, 'sphinx.ext.autodoc')
235        check_sphinx_extension(ctx, 'sphinx.ext.coverage')
236        check_sphinx_extension(ctx, 'sphinx.ext.doctest')
237        check_sphinx_extension(ctx, 'sphinx.ext.graphviz')
238        check_sphinx_extension(ctx, 'sphinx.ext.intersphinx')
239        check_sphinx_extension(ctx, 'sphinx.ext.mathjax')
240        check_sphinx_extension(ctx, 'sphinxcontrib.bibtex')
241
242    #
243    # Optional builds.
244    #
245    ctx.env.BUILD_PDF = 'no'
246    if ctx.options.pdf:
247        if conf.latex_engine == 'xelatex':
248            if not ctx.env.LATEX_CMD:
249                ctx.load('tex')
250                if not ctx.env.XELATEX or not ctx.env.MAKEINDEX:
251                    ctx.fatal('The programs xelatex and makeindex are required for PDF output')
252                ctx.env.LATEX_CMD = 'xelatex'
253                latex.configure_tests(ctx)
254                # Minted needs 'shell-escape'
255                if 'XELATEXFLAGS' not in ctx.env or \
256                   '-shell-escape' not in ctx.env['XELATEXFLAGS']:
257                    ctx.env.append_value('XELATEXFLAGS', '-shell-escape')
258                ctx.env.append_value('MAKEINDEXFLAGS', ['-s', 'python.ist'])
259        elif conf.latex_engine == 'pdflatex':
260            if not ctx.env.LATEX_CMD:
261                ctx.load('tex')
262                if not ctx.env.PDFLATEX or not ctx.env.MAKEINDEX:
263                    ctx.fatal('The programs pdflatex and makeindex are required for PDF output')
264                if 'PDFLATEXFLAGS' not in ctx.env or \
265                   '-shell-escape' not in ctx.env['PDFLATEXFLAGS']:
266                    ctx.env.append_value('PDFLATEXFLAGS', '-shell-escape')
267                ctx.env.append_value('MAKEINDEXFLAGS', ['-s', 'python.ist'])
268                ctx.env.LATEX_CMD = 'pdflatex'
269                latex.configure_tests(ctx)
270        else:
271            ctx.fatal('Unsupported latex engine: %s' % (conf.latex_engine))
272        ctx.env.BUILD_PDF = 'yes'
273
274    ctx.env.BUILD_SINGLEHTML = 'no'
275    if ctx.options.singlehtml:
276        check_inliner = not ctx.env.BIN_INLINER
277        if check_inliner:
278            ctx.env.BUILD_SINGLEHTML = 'yes'
279            ctx.find_program("inliner", var = "BIN_INLINER", mandatory = False)
280            if not ctx.env.BIN_INLINER:
281                ctx.fatal("Node.js inliner is required install with 'npm install -g inliner' " +
282                          "(https://github.com/remy/inliner)")
283
284    ctx.env.BUILD_PLANTUML = 'no'
285    if ctx.options.plantuml:
286        check_plantuml = not ctx.env.BIN_PUML
287        if check_plantuml:
288            ctx.env.BUILD_PLANTUML = 'yes'
289            ctx.find_program("puml", var = "BIN_PUML", mandatory = False)
290            if not ctx.env.BIN_PUML:
291                ctx.fatal("Node.js puml is required install with 'npm install -g node-plantuml' " +
292                          "(https://www.npmjs.com/package/node-plantuml)")
293
294    ctx.env.BUILD_DITAA = 'no'
295    if ctx.options.ditaa:
296        #
297        # We use DITAA via PlantUML
298        #
299        if not ctx.env.BIN_PUML:
300            ctx.find_program("puml", var = "BIN_PUML", mandatory = False)
301            if not ctx.env.BIN_PUML:
302                ctx.fatal("DITAA uses PlantUML; " +
303                          "Node.js puml is required install with 'npm install -g node-plantuml' " +
304                          "(https://www.npmjs.com/package/node-plantuml)")
305        check_ditaa = not ctx.env.BIN_DITAA
306        if check_ditaa:
307            ctx.env.BUILD_DITAA = 'yes'
308            ctx.find_program("ditaa", var = "BIN_DITAA", mandatory = False)
309            if not ctx.env.BIN_DITAA:
310                ctx.fatal("DITAA not found, plase install")
311
312def sources_exclude(ctx, sources):
313    exclude = sources.get('exclude', [])
314    if len(exclude) == 0:
315        return exclude
316    return [ctx.path.find_node(e) for e in exclude]
317
318def sources_extra(ctx, sources):
319    extra = sources.get('extra', [])
320    if len(extra) == 0:
321        extra = [ctx.path.find_node(e) for e in extra]
322    return [e for e in extra if e not in sources_exclude(ctx, sources)]
323
324def sources_source(ctx, sources):
325    extra = sources_extra(ctx, sources)
326    exclude = sources_exclude(ctx, sources)
327    source = ctx.path.ant_glob('**/*.rst')
328    return [s for s in source if s not in exclude] + extra
329
330def doc_pdf(ctx, source_dir, conf_dir, sources):
331    buildtype = 'latex'
332    build_dir, output_node, output_dir, doctrees = build_dir_setup(ctx, buildtype)
333    resources = pdf_resources(ctx, buildtype)
334    rule = sphinx_cmdline(ctx, buildtype, conf_dir, doctrees, source_dir, output_dir)
335    ctx(
336        rule         = rule,
337        cwd          = ctx.path,
338        source       = sources_source(ctx, sources),
339        depends_on   = sources_extra(ctx, sources),
340        target       = ctx.path.find_or_declare("%s/%s.tex" % (buildtype,
341                                                               ctx.path.name))
342    )
343    env_latex = ctx.env.derive()
344    env_latex.TEXINPUTS = output_dir + ':'
345    ctx(
346        features     = 'tex',
347        cwd          = output_dir,
348        type         = ctx.env.LATEX_CMD,
349        source       = "%s/%s.tex" % (buildtype, ctx.path.name),
350        prompt       = 0,
351        env          = env_latex
352    )
353    ctx.install_files('${PREFIX}',
354                      '%s/%s.pdf' % (buildtype, ctx.path.name),
355                      cwd = output_node,
356                      quiet = True)
357
358def doc_singlehtml(ctx, source_dir, conf_dir, sources):
359    #
360    # Use a run command to handle stdout and stderr output from inliner. Using
361    # a standard rule in the build context locks up.
362    #
363    def run(task):
364        src = task.inputs[0].abspath()
365        tgt = task.outputs[0].abspath()
366        cmd = '%s %s' % (task.env.BIN_INLINER[0], src)
367        so = open(tgt, 'w')
368        se = open(tgt + '.err', 'w')
369        r = task.exec_command(cmd, stdout = so, stderr = se)
370        so.close()
371        se.close()
372        #
373        # The inliner does not handle internal href's correctly and places the
374        # input's file name in the href. Strip these.
375        #
376        if sys.version_info[0] < 3:
377            with open(tgt, 'r') as i:
378                before = i.read()
379        else:
380            with open(tgt, 'r', encoding = 'ascii', errors = 'surrogateescape') as i:
381                before = i.read()
382        i.close()
383        after = before.replace('index.html', '')
384        if sys.version_info[0] < 3:
385            with open(tgt, 'w') as o:
386                o.write(after)
387        else:
388            with open(tgt, 'w', encoding = 'ascii', errors = 'surrogateescape') as o:
389                o.write(after)
390        o.close()
391        return r
392
393    buildtype = 'singlehtml'
394    build_dir, output_node, output_dir, doctrees = build_dir_setup(ctx, buildtype)
395    resources = html_resources(ctx, buildtype)
396    rule = sphinx_cmdline(ctx, buildtype, conf_dir, doctrees, source_dir, output_dir)
397    ctx(
398        rule         = rule,
399        cwd          = ctx.path,
400        source       = sources_source(ctx, sources),
401        depends_on   = resources,
402        target       = ctx.path.find_or_declare("%s/index.html" % (buildtype)),
403        install_path = None
404    )
405    ctx(
406        rule         = run,
407        inliner      = ctx.env.BIN_INLINER,
408        source       = "%s/index.html" % buildtype,
409        target       = "%s/%s.html" % (buildtype, ctx.path.name),
410        install_path = '${PREFIX}'
411    )
412
413def doc_html(ctx, source_dir, conf_dir, sources):
414    buildtype = 'html'
415    build_dir, output_node, output_dir, doctrees = build_dir_setup(ctx, buildtype)
416    resources = html_resources(ctx, buildtype)
417    templates = os.path.join(str(ctx.path.get_bld()), buildtype, '_templates')
418    configs = { 'templates_path': templates }
419    rule = sphinx_cmdline(ctx, buildtype, conf_dir, doctrees, source_dir, output_dir, configs)
420    ctx(
421        rule         = rule,
422        cwd          = ctx.path,
423        source       = sources_source(ctx, sources),
424        depends_on   = resources,
425        target       = ctx.path.find_or_declare('%s/index.html' % buildtype),
426        install_path = None
427    )
428    ctx.install_files('${PREFIX}/%s' % (ctx.path.name),
429                      output_node.ant_glob('**/*', quiet = True),
430                      cwd = output_node,
431                      relative_trick = True,
432                      quiet = True)
433
434def images_plantuml(ctx, source_dir, conf_dir, ext):
435    #
436    # Use a run command to handle stdout and stderr output from puml.
437    #
438    def run(task):
439        src = task.inputs[0].abspath()
440        tgt = task.outputs[0].abspath()
441        cmd = '%s generate %s -o %s' % (task.env.BIN_PUML[0], src, tgt)
442        so = open(tgt + '.out', 'w')
443        r = task.exec_command(cmd, stdout = so, stderr = so)
444        so.close()
445        return r
446
447    for src in ctx.path.ant_glob('**/*' + ext):
448        tgt = src.change_ext('.png')
449        ctx(
450            rule         = run,
451            inliner      = ctx.env.BIN_PUML,
452            source       = src,
453            target       = tgt,
454            install_path = None
455        )
456
457def cmd_build(ctx, sources = {}):
458    conf_dir = ctx.path.get_src()
459    source_dir = ctx.path.get_src()
460
461    if ctx.env.BUILD_PDF == 'yes':
462        doc_pdf(ctx, source_dir, conf_dir, sources)
463
464    if ctx.env.BUILD_SINGLEHTML == 'yes':
465        doc_singlehtml(ctx, source_dir, conf_dir, sources)
466
467    doc_html(ctx, source_dir, conf_dir, sources)
468
469def cmd_build_images(ctx):
470    conf_dir = ctx.path.get_src()
471    source_dir = ctx.path.get_src()
472    if ctx.env.BUILD_PLANTUML == 'yes':
473        images_plantuml(ctx, source_dir, conf_dir, '.puml')
474    if ctx.env.BUILD_DITAA == 'yes':
475        images_plantuml(ctx, source_dir, conf_dir, '.ditaa')
476
477def cmd_options(ctx):
478    ctx.add_option('--disable-extra-fonts',
479                   action = 'store_true',
480                   default = False,
481                   help = "Disable building with extra fonts for better quality (lower quality).")
482    ctx.add_option('--sphinx-options',
483                   action = 'store',
484                   default = "",
485                   help = "Additional Sphinx options.")
486    ctx.add_option('--sphinx-nit-pick',
487                   action = 'store_true',
488                   default = False,
489                   help = "Enable Sphinx nit-picky mode else be quiet")
490    ctx.add_option('--pdf',
491                   action = 'store_true',
492                   default = False,
493                   help = "Build PDF.")
494    ctx.add_option('--singlehtml',
495                   action = 'store_true',
496                   default = False,
497                   help = "Build Single HTML file, requires Node Inliner")
498    ctx.add_option('--plantuml',
499                   action = 'store_true',
500                   default = False,
501                   help = "Build PlantUML images from source, need puml from npm")
502    ctx.add_option('--ditaa',
503                   action = 'store_true',
504                   default = False,
505                   help = "Build DITAA images using PlantUML from source, need ditaa and puml")
506
507def cmd_options_path(ctx):
508    cmd_options(ctx)
509    ctx.add_option('--rtems-path-py',
510                   type = 'string',
511                   help = "Full path to py/ in RTEMS source repository.")
512
513def cmd_configure_path(ctx):
514    if not ctx.options.rtems_path_py:
515        ctx.fatal("--rtems-path-py is required")
516
517    ctx.env.RTEMS_PATH = ctx.options.rtems_path_py
518
519    cmd_configure(ctx)
520
521def xml_catalogue(ctx, building):
522    #
523    # The following is a hack to find the top_dir because the task does
524    # provided a reference to top_dir like a build context.
525    #
526    top_dir = ctx.get_cwd().find_node('..')
527    #
528    # Read the conf.py files in each directory to gather the doc details.
529    #
530    catalogue = {}
531    sp = sys.path[:]
532    for doc in building:
533        #
534        # Import using the imp API so the module is reloaded for us.
535        #
536        import imp
537        sys.path = [top_dir.find_node(doc).abspath()]
538        mf = imp.find_module('conf')
539        sys.path = sp[:]
540        try:
541            bconf = imp.load_module('bconf', mf[0], mf[1], mf[2])
542        finally:
543            mf[0].close()
544        catalogue[doc] = {
545            'title': bconf.project,
546            'version': str(ctx.env.VERSION),
547            'release': str(ctx.env.VERSION),
548            'pdf': bconf.latex_documents[0][1].replace('.tex', '.pdf'),
549            'html': '%s/index.html' % (doc),
550            'singlehtml': '%s.html' % (doc)
551        }
552        bconf = None
553
554    import xml.dom.minidom as xml
555    cat = xml.Document()
556
557    root = cat.createElement('rtems-docs')
558    root.setAttribute('date', ctx.env.DATE)
559    cat.appendChild(root)
560
561    heading = cat.createElement('catalogue')
562    text = cat.createTextNode(str(ctx.env.VERSION))
563    heading.appendChild(text)
564    root.appendChild(heading)
565
566    builds = ['html']
567    if ctx.env.BUILD_PDF == 'yes':
568        builds += ['pdf']
569    if ctx.env.BUILD_SINGLEHTML == 'yes':
570        builds += ['singlehtml']
571
572    for d in building:
573        doc = cat.createElement('doc')
574        name = cat.createElement('name')
575        text = cat.createTextNode(d)
576        name.appendChild(text)
577        title = cat.createElement('title')
578        text = cat.createTextNode(catalogue[d]['title'])
579        title.appendChild(text)
580        release = cat.createElement('release')
581        text = cat.createTextNode(catalogue[d]['release'])
582        release.appendChild(text)
583        version = cat.createElement('version')
584        text = cat.createTextNode(catalogue[d]['version'])
585        version.appendChild(text)
586        doc.appendChild(name)
587        doc.appendChild(title)
588        doc.appendChild(release)
589        doc.appendChild(version)
590        for b in builds:
591            output = cat.createElement(b)
592            text = cat.createTextNode(catalogue[d][b])
593            output.appendChild(text)
594            doc.appendChild(output)
595        root.appendChild(doc)
596
597    catnode = ctx.get_cwd().make_node('catalogue.xml')
598    catnode.write(cat.toprettyxml(indent = ' ' * 2, newl = os.linesep))
599
600    cat.unlink()
Note: See TracBrowser for help on using the repository browser.