Context Navigation

source: rtems-tools/doc/asciidoc/asciidoc.py @ f91e023

4.104.115

Last change on this file since f91e023 was f91e023, checked in by Chris Johns <chrisj@…>, on 02/17/14 at 07:04:46
Add the documentation.
Property mode set to `100755`
File size: 247.9 KB

Line
1	#!/usr/bin/env python
2	"""
3	asciidoc - converts an AsciiDoc text file to HTML or DocBook
4
5	Copyright (C) 2002-2010 Stuart Rackham. Free use of this software is granted
6	under the terms of the GNU General Public License (GPL).
7	"""
8
9	import sys, os, re, time, traceback, tempfile, subprocess, codecs, locale, unicodedata, copy
10
11	### Used by asciidocapi.py ###
12	VERSION = '8.6.8' # See CHANGLOG file for version history.
13
14	MIN_PYTHON_VERSION = '2.4' # Require this version of Python or better.
15
16	#---------------------------------------------------------------------------
17	# Program constants.
18	#---------------------------------------------------------------------------
19	DEFAULT_BACKEND = 'html'
20	DEFAULT_DOCTYPE = 'article'
21	# Allowed substitution options for List, Paragraph and DelimitedBlock
22	# definition subs entry.
23	SUBS_OPTIONS = ('specialcharacters','quotes','specialwords',
24	'replacements', 'attributes','macros','callouts','normal','verbatim',
25	'none','replacements2','replacements3')
26	# Default value for unspecified subs and presubs configuration file entries.
27	SUBS_NORMAL = ('specialcharacters','quotes','attributes',
28	'specialwords','replacements','macros','replacements2')
29	SUBS_VERBATIM = ('specialcharacters','callouts')
30
31	NAME_RE = r'(?u)[^\W\d][-\w]*' # Valid section or attribute name.
32	OR, AND = ',', '+' # Attribute list separators.
33
34
35	#---------------------------------------------------------------------------
36	# Utility functions and classes.
37	#---------------------------------------------------------------------------
38
39	class EAsciiDoc(Exception): pass
40
41	class OrderedDict(dict):
42	"""
43	Dictionary ordered by insertion order.
44	Python Cookbook: Ordered Dictionary, Submitter: David Benjamin.
45	http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/107747
46	"""
47	def __init__(self, d=None, **kwargs):
48	self._keys = []
49	if d is None: d = kwargs
50	dict.__init__(self, d)
51	def __delitem__(self, key):
52	dict.__delitem__(self, key)
53	self._keys.remove(key)
54	def __setitem__(self, key, item):
55	dict.__setitem__(self, key, item)
56	if key not in self._keys: self._keys.append(key)
57	def clear(self):
58	dict.clear(self)
59	self._keys = []
60	def copy(self):
61	d = dict.copy(self)
62	d._keys = self._keys[:]
63	return d
64	def items(self):
65	return zip(self._keys, self.values())
66	def keys(self):
67	return self._keys
68	def popitem(self):
69	try:
70	key = self._keys[-1]
71	except IndexError:
72	raise KeyError('dictionary is empty')
73	val = self[key]
74	del self[key]
75	return (key, val)
76	def setdefault(self, key, failobj = None):
77	dict.setdefault(self, key, failobj)
78	if key not in self._keys: self._keys.append(key)
79	def update(self, d=None, **kwargs):
80	if d is None:
81	d = kwargs
82	dict.update(self, d)
83	for key in d.keys():
84	if key not in self._keys: self._keys.append(key)
85	def values(self):
86	return map(self.get, self._keys)
87
88	class AttrDict(dict):
89	"""
90	Like a dictionary except values can be accessed as attributes i.e. obj.foo
91	can be used in addition to obj['foo'].
92	If an item is not present None is returned.
93	"""
94	def __getattr__(self, key):
95	try: return self[key]
96	except KeyError: return None
97	def __setattr__(self, key, value):
98	self[key] = value
99	def __delattr__(self, key):
100	try: del self[key]
101	except KeyError, k: raise AttributeError, k
102	def __repr__(self):
103	return '<AttrDict ' + dict.__repr__(self) + '>'
104	def __getstate__(self):
105	return dict(self)
106	def __setstate__(self,value):
107	for k,v in value.items(): self[k]=v
108
109	class InsensitiveDict(dict):
110	"""
111	Like a dictionary except key access is case insensitive.
112	Keys are stored in lower case.
113	"""
114	def __getitem__(self, key):
115	return dict.__getitem__(self, key.lower())
116	def __setitem__(self, key, value):
117	dict.__setitem__(self, key.lower(), value)
118	def has_key(self, key):
119	return dict.has_key(self,key.lower())
120	def get(self, key, default=None):
121	return dict.get(self, key.lower(), default)
122	def update(self, dict):
123	for k,v in dict.items():
124	self[k] = v
125	def setdefault(self, key, default = None):
126	return dict.setdefault(self, key.lower(), default)
127
128
129	class Trace(object):
130	"""
131	Used in conjunction with the 'trace' attribute to generate diagnostic
132	output. There is a single global instance of this class named trace.
133	"""
134	SUBS_NAMES = ('specialcharacters','quotes','specialwords',
135	'replacements', 'attributes','macros','callouts',
136	'replacements2','replacements3')
137	def __init__(self):
138	self.name_re = '' # Regexp pattern to match trace names.
139	self.linenos = True
140	self.offset = 0
141	def __call__(self, name, before, after=None):
142	"""
143	Print trace message if tracing is on and the trace 'name' matches the
144	document 'trace' attribute (treated as a regexp).
145	'before' is the source text before substitution; 'after' text is the
146	source text after substitutuion.
147	The 'before' and 'after' messages are only printed if they differ.
148	"""
149	name_re = document.attributes.get('trace')
150	if name_re == 'subs': # Alias for all the inline substitutions.
151	name_re = '\|'.join(self.SUBS_NAMES)
152	self.name_re = name_re
153	if self.name_re is not None:
154	msg = message.format(name, 'TRACE: ', self.linenos, offset=self.offset)
155	if before != after and re.match(self.name_re,name):
156	if is_array(before):
157	before = '\n'.join(before)
158	if after is None:
159	msg += '\n%s\n' % before
160	else:
161	if is_array(after):
162	after = '\n'.join(after)
163	msg += '\n<<<\n%s\n>>>\n%s\n' % (before,after)
164	message.stderr(msg)
165
166	class Message:
167	"""
168	Message functions.
169	"""
170	PROG = os.path.basename(os.path.splitext(__file__)[0])
171
172	def __init__(self):
173	# Set to True or False to globally override line numbers method
174	# argument. Has no effect when set to None.
175	self.linenos = None
176	self.messages = []
177	self.prev_msg = ''
178
179	def stdout(self,msg):
180	print msg
181
182	def stderr(self,msg=''):
183	if msg == self.prev_msg: # Suppress repeated messages.
184	return
185	self.messages.append(msg)
186	if __name__ == '__main__':
187	sys.stderr.write('%s: %s%s' % (self.PROG, msg, os.linesep))
188	self.prev_msg = msg
189
190	def verbose(self, msg,linenos=True):
191	if config.verbose:
192	msg = self.format(msg,linenos=linenos)
193	self.stderr(msg)
194
195	def warning(self, msg,linenos=True,offset=0):
196	msg = self.format(msg,'WARNING: ',linenos,offset=offset)
197	document.has_warnings = True
198	self.stderr(msg)
199
200	def deprecated(self, msg, linenos=True):
201	msg = self.format(msg, 'DEPRECATED: ', linenos)
202	self.stderr(msg)
203
204	def format(self, msg, prefix='', linenos=True, cursor=None, offset=0):
205	"""Return formatted message string."""
206	if self.linenos is not False and ((linenos or self.linenos) and reader.cursor):
207	if cursor is None:
208	cursor = reader.cursor
209	prefix += '%s: line %d: ' % (os.path.basename(cursor[0]),cursor[1]+offset)
210	return prefix + msg
211
212	def error(self, msg, cursor=None, halt=False):
213	"""
214	Report fatal error.
215	If halt=True raise EAsciiDoc exception.
216	If halt=False don't exit application, continue in the hope of reporting
217	all fatal errors finishing with a non-zero exit code.
218	"""
219	if halt:
220	raise EAsciiDoc, self.format(msg,linenos=False,cursor=cursor)
221	else:
222	msg = self.format(msg,'ERROR: ',cursor=cursor)
223	self.stderr(msg)
224	document.has_errors = True
225
226	def unsafe(self, msg):
227	self.error('unsafe: '+msg)
228
229
230	def userdir():
231	"""
232	Return user's home directory or None if it is not defined.
233	"""
234	result = os.path.expanduser('~')
235	if result == '~':
236	result = None
237	return result
238
239	def localapp():
240	"""
241	Return True if we are not executing the system wide version
242	i.e. the configuration is in the executable's directory.
243	"""
244	return os.path.isfile(os.path.join(APP_DIR, 'asciidoc.conf'))
245
246	def file_in(fname, directory):
247	"""Return True if file fname resides inside directory."""
248	assert os.path.isfile(fname)
249	# Empty directory (not to be confused with None) is the current directory.
250	if directory == '':
251	directory = os.getcwd()
252	else:
253	assert os.path.isdir(directory)
254	directory = os.path.realpath(directory)
255	fname = os.path.realpath(fname)
256	return os.path.commonprefix((directory, fname)) == directory
257
258	def safe():
259	return document.safe
260
261	def is_safe_file(fname, directory=None):
262	# A safe file must reside in 'directory' (defaults to the source
263	# file directory).
264	if directory is None:
265	if document.infile == '<stdin>':
266	return not safe()
267	directory = os.path.dirname(document.infile)
268	elif directory == '':
269	directory = '.'
270	return (
271	not safe()
272	or file_in(fname, directory)
273	or file_in(fname, APP_DIR)
274	or file_in(fname, CONF_DIR)
275	)
276
277	def safe_filename(fname, parentdir):
278	"""
279	Return file name which must reside in the parent file directory.
280	Return None if file is not safe.
281	"""
282	if not os.path.isabs(fname):
283	# Include files are relative to parent document
284	# directory.
285	fname = os.path.normpath(os.path.join(parentdir,fname))
286	if not is_safe_file(fname, parentdir):
287	message.unsafe('include file: %s' % fname)
288	return None
289	return fname
290
291	def assign(dst,src):
292	"""Assign all attributes from 'src' object to 'dst' object."""
293	for a,v in src.__dict__.items():
294	setattr(dst,a,v)
295
296	def strip_quotes(s):
297	"""Trim white space and, if necessary, quote characters from s."""
298	s = s.strip()
299	# Strip quotation mark characters from quoted strings.
300	if len(s) >= 3 and s[0] == '"' and s[-1] == '"':
301	s = s[1:-1]
302	return s
303
304	def is_re(s):
305	"""Return True if s is a valid regular expression else return False."""
306	try: re.compile(s)
307	except: return False
308	else: return True
309
310	def re_join(relist):
311	"""Join list of regular expressions re1,re2,... to single regular
312	expression (re1)\|(re2)\|..."""
313	if len(relist) == 0:
314	return None
315	result = []
316	# Delete named groups to avoid ambiguity.
317	for s in relist:
318	result.append(re.sub(r'\?P<\S+?>','',s))
319	result = ')\|('.join(result)
320	result = '('+result+')'
321	return result
322
323	def lstrip_list(s):
324	"""
325	Return list with empty items from start of list removed.
326	"""
327	for i in range(len(s)):
328	if s[i]: break
329	else:
330	return []
331	return s[i:]
332
333	def rstrip_list(s):
334	"""
335	Return list with empty items from end of list removed.
336	"""
337	for i in range(len(s)-1,-1,-1):
338	if s[i]: break
339	else:
340	return []
341	return s[:i+1]
342
343	def strip_list(s):
344	"""
345	Return list with empty items from start and end of list removed.
346	"""
347	s = lstrip_list(s)
348	s = rstrip_list(s)
349	return s
350
351	def is_array(obj):
352	"""
353	Return True if object is list or tuple type.
354	"""
355	return isinstance(obj,list) or isinstance(obj,tuple)
356
357	def dovetail(lines1, lines2):
358	"""
359	Append list or tuple of strings 'lines2' to list 'lines1'. Join the last
360	non-blank item in 'lines1' with the first non-blank item in 'lines2' into a
361	single string.
362	"""
363	assert is_array(lines1)
364	assert is_array(lines2)
365	lines1 = strip_list(lines1)
366	lines2 = strip_list(lines2)
367	if not lines1 or not lines2:
368	return list(lines1) + list(lines2)
369	result = list(lines1[:-1])
370	result.append(lines1[-1] + lines2[0])
371	result += list(lines2[1:])
372	return result
373
374	def dovetail_tags(stag,content,etag):
375	"""Merge the end tag with the first content line and the last
376	content line with the end tag. This ensures verbatim elements don't
377	include extraneous opening and closing line breaks."""
378	return dovetail(dovetail(stag,content), etag)
379
380	# The following functions are so we don't have to use the dangerous
381	# built-in eval() function.
382	if float(sys.version[:3]) >= 2.6 or sys.platform[:4] == 'java':
383	# Use AST module if CPython >= 2.6 or Jython.
384	import ast
385	from ast import literal_eval
386
387	def get_args(val):
388	d = {}
389	args = ast.parse("d(" + val + ")", mode='eval').body.args
390	i = 1
391	for arg in args:
392	if isinstance(arg, ast.Name):
393	d[str(i)] = literal_eval(arg.id)
394	else:
395	d[str(i)] = literal_eval(arg)
396	i += 1
397	return d
398
399	def get_kwargs(val):
400	d = {}
401	args = ast.parse("d(" + val + ")", mode='eval').body.keywords
402	for arg in args:
403	d[arg.arg] = literal_eval(arg.value)
404	return d
405
406	def parse_to_list(val):
407	values = ast.parse("[" + val + "]", mode='eval').body.elts
408	return [literal_eval(v) for v in values]
409
410	else: # Use deprecated CPython compiler module.
411	import compiler
412	from compiler.ast import Const, Dict, Expression, Name, Tuple, UnarySub, Keyword
413
414	# Code from:
415	# http://mail.python.org/pipermail/python-list/2009-September/1219992.html
416	# Modified to use compiler.ast.List as this module has a List
417	def literal_eval(node_or_string):
418	"""
419	Safely evaluate an expression node or a string containing a Python
420	expression. The string or node provided may only consist of the
421	following Python literal structures: strings, numbers, tuples,
422	lists, dicts, booleans, and None.
423	"""
424	_safe_names = {'None': None, 'True': True, 'False': False}
425	if isinstance(node_or_string, basestring):
426	node_or_string = compiler.parse(node_or_string, mode='eval')
427	if isinstance(node_or_string, Expression):
428	node_or_string = node_or_string.node
429	def _convert(node):
430	if isinstance(node, Const) and isinstance(node.value,
431	(basestring, int, float, long, complex)):
432	return node.value
433	elif isinstance(node, Tuple):
434	return tuple(map(_convert, node.nodes))
435	elif isinstance(node, compiler.ast.List):
436	return list(map(_convert, node.nodes))
437	elif isinstance(node, Dict):
438	return dict((_convert(k), _convert(v)) for k, v
439	in node.items)
440	elif isinstance(node, Name):
441	if node.name in _safe_names:
442	return _safe_names[node.name]
443	elif isinstance(node, UnarySub):
444	return -_convert(node.expr)
445	raise ValueError('malformed string')
446	return _convert(node_or_string)
447
448	def get_args(val):
449	d = {}
450	args = compiler.parse("d(" + val + ")", mode='eval').node.args
451	i = 1
452	for arg in args:
453	if isinstance(arg, Keyword):
454	break
455	d[str(i)] = literal_eval(arg)
456	i = i + 1
457	return d
458
459	def get_kwargs(val):
460	d = {}
461	args = compiler.parse("d(" + val + ")", mode='eval').node.args
462	i = 0
463	for arg in args:
464	if isinstance(arg, Keyword):
465	break
466	i += 1
467	args = args[i:]
468	for arg in args:
469	d[str(arg.name)] = literal_eval(arg.expr)
470	return d
471
472	def parse_to_list(val):
473	values = compiler.parse("[" + val + "]", mode='eval').node.asList()
474	return [literal_eval(v) for v in values]
475
476	def parse_attributes(attrs,dict):
477	"""Update a dictionary with name/value attributes from the attrs string.
478	The attrs string is a comma separated list of values and keyword name=value
479	pairs. Values must preceed keywords and are named '1','2'... The entire
480	attributes list is named '0'. If keywords are specified string values must
481	be quoted. Examples:
482
483	attrs: ''
484	dict: {}
485
486	attrs: 'hello,world'
487	dict: {'2': 'world', '0': 'hello,world', '1': 'hello'}
488
489	attrs: '"hello", planet="earth"'
490	dict: {'planet': 'earth', '0': '"hello",planet="earth"', '1': 'hello'}
491	"""
492	def f(args,*keywords):
493	# Name and add aguments '1','2'... to keywords.
494	for i in range(len(args)):
495	if not str(i+1) in keywords:
496	keywords[str(i+1)] = args[i]
497	return keywords
498
499	if not attrs:
500	return
501	dict['0'] = attrs
502	# Replace line separators with spaces so line spanning works.
503	s = re.sub(r'\s', ' ', attrs)
504	d = {}
505	try:
506	d.update(get_args(s))
507	d.update(get_kwargs(s))
508	for v in d.values():
509	if not (isinstance(v,str) or isinstance(v,int) or isinstance(v,float) or v is None):
510	raise Exception
511	except Exception:
512	s = s.replace('"','\\"')
513	s = s.split(',')
514	s = map(lambda x: '"' + x.strip() + '"', s)
515	s = ','.join(s)
516	try:
517	d = {}
518	d.update(get_args(s))
519	d.update(get_kwargs(s))
520	except Exception:
521	return # If there's a syntax error leave with {0}=attrs.
522	for k in d.keys(): # Drop any empty positional arguments.
523	if d[k] == '': del d[k]
524	dict.update(d)
525	assert len(d) > 0
526
527	def parse_named_attributes(s,attrs):
528	"""Update a attrs dictionary with name="value" attributes from the s string.
529	Returns False if invalid syntax.
530	Example:
531	attrs: 'star="sun",planet="earth"'
532	dict: {'planet':'earth', 'star':'sun'}
533	"""
534	def f(**keywords): return keywords
535
536	try:
537	d = {}
538	d = get_kwargs(s)
539	attrs.update(d)
540	return True
541	except Exception:
542	return False
543
544	def parse_list(s):
545	"""Parse comma separated string of Python literals. Return a tuple of of
546	parsed values."""
547	try:
548	result = tuple(parse_to_list(s))
549	except Exception:
550	raise EAsciiDoc,'malformed list: '+s
551	return result
552
553	def parse_options(options,allowed,errmsg):
554	"""Parse comma separated string of unquoted option names and return as a
555	tuple of valid options. 'allowed' is a list of allowed option values.
556	If allowed=() then all legitimate names are allowed.
557	'errmsg' is an error message prefix if an illegal option error is thrown."""
558	result = []
559	if options:
560	for s in re.split(r'\s,\s',options):
561	if (allowed and s not in allowed) or not is_name(s):
562	raise EAsciiDoc,'%s: %s' % (errmsg,s)
563	result.append(s)
564	return tuple(result)
565
566	def symbolize(s):
567	"""Drop non-symbol characters and convert to lowercase."""
568	return re.sub(r'(?u)[^\w\-_]', '', s).lower()
569
570	def is_name(s):
571	"""Return True if s is valid attribute, macro or tag name
572	(starts with alpha containing alphanumeric and dashes only)."""
573	return re.match(r'^'+NAME_RE+r'$',s) is not None
574
575	def subs_quotes(text):
576	"""Quoted text is marked up and the resulting text is
577	returned."""
578	keys = config.quotes.keys()
579	for q in keys:
580	i = q.find('\|')
581	if i != -1 and q != '\|' and q != '\|\|':
582	lq = q[:i] # Left quote.
583	rq = q[i+1:] # Right quote.
584	else:
585	lq = rq = q
586	tag = config.quotes[q]
587	if not tag: continue
588	# Unconstrained quotes prefix the tag name with a hash.
589	if tag[0] == '#':
590	tag = tag[1:]
591	# Unconstrained quotes can appear anywhere.
592	reo = re.compile(r'(?msu)(^\|.)(\[(?P<attrlist>[^[\]]+?)\])?' \
593	+ r'(?:' + re.escape(lq) + r')' \
594	+ r'(?P<content>.+?)(?:'+re.escape(rq)+r')')
595	else:
596	# The text within constrained quotes must be bounded by white space.
597	# Non-word (\W) characters are allowed at boundaries to accomodate
598	# enveloping quotes and punctuation e.g. a='x', ('x'), 'x', ['x'].
599	reo = re.compile(r'(?msu)(^\|[^\w;:}])(\[(?P<attrlist>[^[\]]+?)\])?' \
600	+ r'(?:' + re.escape(lq) + r')' \
601	+ r'(?P<content>\S\|\S.*?\S)(?:'+re.escape(rq)+r')(?=\W\|$)')
602	pos = 0
603	while True:
604	mo = reo.search(text,pos)
605	if not mo: break
606	if text[mo.start()] == '\\':
607	# Delete leading backslash.
608	text = text[:mo.start()] + text[mo.start()+1:]
609	# Skip past start of match.
610	pos = mo.start() + 1
611	else:
612	attrlist = {}
613	parse_attributes(mo.group('attrlist'), attrlist)
614	stag,etag = config.tag(tag, attrlist)
615	s = mo.group(1) + stag + mo.group('content') + etag
616	text = text[:mo.start()] + s + text[mo.end():]
617	pos = mo.start() + len(s)
618	return text
619
620	def subs_tag(tag,dict={}):
621	"""Perform attribute substitution and split tag string returning start, end
622	tag tuple (c.f. Config.tag())."""
623	if not tag:
624	return [None,None]
625	s = subs_attrs(tag,dict)
626	if not s:
627	message.warning('tag \'%s\' dropped: contains undefined attribute' % tag)
628	return [None,None]
629	result = s.split('\|')
630	if len(result) == 1:
631	return result+[None]
632	elif len(result) == 2:
633	return result
634	else:
635	raise EAsciiDoc,'malformed tag: %s' % tag
636
637	def parse_entry(entry, dict=None, unquote=False, unique_values=False,
638	allow_name_only=False, escape_delimiter=True):
639	"""Parse name=value entry to dictionary 'dict'. Return tuple (name,value)
640	or None if illegal entry.
641	If name= then value is set to ''.
642	If name and allow_name_only=True then value is set to ''.
643	If name! and allow_name_only=True then value is set to None.
644	Leading and trailing white space is striped from 'name' and 'value'.
645	'name' can contain any printable characters.
646	If the '=' delimiter character is allowed in the 'name' then
647	it must be escaped with a backslash and escape_delimiter must be True.
648	If 'unquote' is True leading and trailing double-quotes are stripped from
649	'name' and 'value'.
650	If unique_values' is True then dictionary entries with the same value are
651	removed before the parsed entry is added."""
652	if escape_delimiter:
653	mo = re.search(r'(?:[^\\](=))',entry)
654	else:
655	mo = re.search(r'(=)',entry)
656	if mo: # name=value entry.
657	if mo.group(1):
658	name = entry[:mo.start(1)]
659	if escape_delimiter:
660	name = name.replace(r'\=','=') # Unescape \= in name.
661	value = entry[mo.end(1):]
662	elif allow_name_only and entry: # name or name! entry.
663	name = entry
664	if name[-1] == '!':
665	name = name[:-1]
666	value = None
667	else:
668	value = ''
669	else:
670	return None
671	if unquote:
672	name = strip_quotes(name)
673	if value is not None:
674	value = strip_quotes(value)
675	else:
676	name = name.strip()
677	if value is not None:
678	value = value.strip()
679	if not name:
680	return None
681	if dict is not None:
682	if unique_values:
683	for k,v in dict.items():
684	if v == value: del dict[k]
685	dict[name] = value
686	return name,value
687
688	def parse_entries(entries, dict, unquote=False, unique_values=False,
689	allow_name_only=False,escape_delimiter=True):
690	"""Parse name=value entries from from lines of text in 'entries' into
691	dictionary 'dict'. Blank lines are skipped."""
692	entries = config.expand_templates(entries)
693	for entry in entries:
694	if entry and not parse_entry(entry, dict, unquote, unique_values,
695	allow_name_only, escape_delimiter):
696	raise EAsciiDoc,'malformed section entry: %s' % entry
697
698	def dump_section(name,dict,f=sys.stdout):
699	"""Write parameters in 'dict' as in configuration file section format with
700	section 'name'."""
701	f.write('[%s]%s' % (name,writer.newline))
702	for k,v in dict.items():
703	k = str(k)
704	k = k.replace('=',r'\=') # Escape = in name.
705	# Quote if necessary.
706	if len(k) != len(k.strip()):
707	k = '"'+k+'"'
708	if v and len(v) != len(v.strip()):
709	v = '"'+v+'"'
710	if v is None:
711	# Don't dump undefined attributes.
712	continue
713	else:
714	s = k+'='+v
715	if s[0] == '#':
716	s = '\\' + s # Escape so not treated as comment lines.
717	f.write('%s%s' % (s,writer.newline))
718	f.write(writer.newline)
719
720	def update_attrs(attrs,dict):
721	"""Update 'attrs' dictionary with parsed attributes in dictionary 'dict'."""
722	for k,v in dict.items():
723	if not is_name(k):
724	raise EAsciiDoc,'illegal attribute name: %s' % k
725	attrs[k] = v
726
727	def is_attr_defined(attrs,dic):
728	"""
729	Check if the sequence of attributes is defined in dictionary 'dic'.
730	Valid 'attrs' sequence syntax:
731	<attr> Return True if single attrbiute is defined.
732	<attr1>,<attr2>,... Return True if one or more attributes are defined.
733	<attr1>+<attr2>+... Return True if all the attributes are defined.
734	"""
735	if OR in attrs:
736	for a in attrs.split(OR):
737	if dic.get(a.strip()) is not None:
738	return True
739	else: return False
740	elif AND in attrs:
741	for a in attrs.split(AND):
742	if dic.get(a.strip()) is None:
743	return False
744	else: return True
745	else:
746	return dic.get(attrs.strip()) is not None
747
748	def filter_lines(filter_cmd, lines, attrs={}):
749	"""
750	Run 'lines' through the 'filter_cmd' shell command and return the result.
751	The 'attrs' dictionary contains additional filter attributes.
752	"""
753	def findfilter(name,dir,filter):
754	"""Find filter file 'fname' with style name 'name' in directory
755	'dir'. Return found file path or None if not found."""
756	if name:
757	result = os.path.join(dir,'filters',name,filter)
758	if os.path.isfile(result):
759	return result
760	result = os.path.join(dir,'filters',filter)
761	if os.path.isfile(result):
762	return result
763	return None
764
765	# Return input lines if there's not filter.
766	if not filter_cmd or not filter_cmd.strip():
767	return lines
768	# Perform attributes substitution on the filter command.
769	s = subs_attrs(filter_cmd, attrs)
770	if not s:
771	message.error('undefined filter attribute in command: %s' % filter_cmd)
772	return []
773	filter_cmd = s.strip()
774	# Parse for quoted and unquoted command and command tail.
775	# Double quoted.
776	mo = re.match(r'^"(?P<cmd>[^"]+)"(?P<tail>.*)$', filter_cmd)
777	if not mo:
778	# Single quoted.
779	mo = re.match(r"^'(?P<cmd>[^']+)'(?P<tail>.*)$", filter_cmd)
780	if not mo:
781	# Unquoted catch all.
782	mo = re.match(r'^(?P<cmd>\S+)(?P<tail>.*)$', filter_cmd)
783	cmd = mo.group('cmd').strip()
784	found = None
785	if not os.path.dirname(cmd):
786	# Filter command has no directory path so search filter directories.
787	filtername = attrs.get('style')
788	d = document.attributes.get('docdir')
789	if d:
790	found = findfilter(filtername, d, cmd)
791	if not found:
792	if USER_DIR:
793	found = findfilter(filtername, USER_DIR, cmd)
794	if not found:
795	if localapp():
796	found = findfilter(filtername, APP_DIR, cmd)
797	else:
798	found = findfilter(filtername, CONF_DIR, cmd)
799	else:
800	if os.path.isfile(cmd):
801	found = cmd
802	else:
803	message.warning('filter not found: %s' % cmd)
804	if found:
805	filter_cmd = '"' + found + '"' + mo.group('tail')
806	if found:
807	if cmd.endswith('.py'):
808	filter_cmd = '"%s" %s' % (document.attributes['python'],
809	filter_cmd)
810	elif cmd.endswith('.rb'):
811	filter_cmd = 'ruby ' + filter_cmd
812
813	message.verbose('filtering: ' + filter_cmd)
814	if os.name == 'nt':
815	# Remove redundant quoting -- this is not just
816	# cosmetic, unnecessary quoting appears to cause
817	# command line truncation.
818	filter_cmd = re.sub(r'"([^ ]+?)"', r'\1', filter_cmd)
819	try:
820	p = subprocess.Popen(filter_cmd, shell=True,
821	stdin=subprocess.PIPE, stdout=subprocess.PIPE)
822	output = p.communicate(os.linesep.join(lines))[0]
823	except Exception:
824	raise EAsciiDoc,'filter error: %s: %s' % (filter_cmd, sys.exc_info()[1])
825	if output:
826	result = [s.rstrip() for s in output.split(os.linesep)]
827	else:
828	result = []
829	filter_status = p.wait()
830	if filter_status:
831	message.warning('filter non-zero exit code: %s: returned %d' %
832	(filter_cmd, filter_status))
833	if lines and not result:
834	message.warning('no output from filter: %s' % filter_cmd)
835	return result
836
837	def system(name, args, is_macro=False, attrs=None):
838	"""
839	Evaluate a system attribute ({name:args}) or system block macro
840	(name::[args]).
841	If is_macro is True then we are processing a system block macro otherwise
842	it's a system attribute.
843	The attrs dictionary is updated by the counter and set system attributes.
844	NOTE: The include1 attribute is used internally by the include1::[] macro
845	and is not for public use.
846	"""
847	if is_macro:
848	syntax = '%s::[%s]' % (name,args)
849	separator = '\n'
850	else:
851	syntax = '{%s:%s}' % (name,args)
852	separator = writer.newline
853	if name not in ('eval','eval3','sys','sys2','sys3','include','include1','counter','counter2','set','set2','template'):
854	if is_macro:
855	msg = 'illegal system macro name: %s' % name
856	else:
857	msg = 'illegal system attribute name: %s' % name
858	message.warning(msg)
859	return None
860	if is_macro:
861	s = subs_attrs(args)
862	if s is None:
863	message.warning('skipped %s: undefined attribute in: %s' % (name,args))
864	return None
865	args = s
866	if name != 'include1':
867	message.verbose('evaluating: %s' % syntax)
868	if safe() and name not in ('include','include1'):
869	message.unsafe(syntax)
870	return None
871	result = None
872	if name in ('eval','eval3'):
873	try:
874	result = eval(args)
875	if result is True:
876	result = ''
877	elif result is False:
878	result = None
879	elif result is not None:
880	result = str(result)
881	except Exception:
882	message.warning('%s: evaluation error' % syntax)
883	elif name in ('sys','sys2','sys3'):
884	result = ''
885	fd,tmp = tempfile.mkstemp()
886	os.close(fd)
887	try:
888	cmd = args
889	cmd = cmd + (' > "%s"' % tmp)
890	if name == 'sys2':
891	cmd = cmd + ' 2>&1'
892	if os.name == 'nt':
893	# Remove redundant quoting -- this is not just
894	# cosmetic, unnecessary quoting appears to cause
895	# command line truncation.
896	cmd = re.sub(r'"([^ ]+?)"', r'\1', cmd)
897	message.verbose('shelling: %s' % cmd)
898	if os.system(cmd):
899	message.warning('%s: non-zero exit status' % syntax)
900	try:
901	if os.path.isfile(tmp):
902	f = open(tmp)
903	try:
904	lines = [s.rstrip() for s in f]
905	finally:
906	f.close()
907	else:
908	lines = []
909	except Exception:
910	raise EAsciiDoc,'%s: temp file read error' % syntax
911	result = separator.join(lines)
912	finally:
913	if os.path.isfile(tmp):
914	os.remove(tmp)
915	elif name in ('counter','counter2'):
916	mo = re.match(r'^(?P<attr>[^:]?)(:(?P<seed>.))?$', args)
917	attr = mo.group('attr')
918	seed = mo.group('seed')
919	if seed and (not re.match(r'^\d+$', seed) and len(seed) > 1):
920	message.warning('%s: illegal counter seed: %s' % (syntax,seed))
921	return None
922	if not is_name(attr):
923	message.warning('%s: illegal attribute name' % syntax)
924	return None
925	value = document.attributes.get(attr)
926	if value:
927	if not re.match(r'^\d+$', value) and len(value) > 1:
928	message.warning('%s: illegal counter value: %s'
929	% (syntax,value))
930	return None
931	if re.match(r'^\d+$', value):
932	expr = value + '+1'
933	else:
934	expr = 'chr(ord("%s")+1)' % value
935	try:
936	result = str(eval(expr))
937	except Exception:
938	message.warning('%s: evaluation error: %s' % (syntax, expr))
939	else:
940	if seed:
941	result = seed
942	else:
943	result = '1'
944	document.attributes[attr] = result
945	if attrs is not None:
946	attrs[attr] = result
947	if name == 'counter2':
948	result = ''
949	elif name in ('set','set2'):
950	mo = re.match(r'^(?P<attr>[^:]?)(:(?P<value>.))?$', args)
951	attr = mo.group('attr')
952	value = mo.group('value')
953	if value is None:
954	value = ''
955	if attr.endswith('!'):
956	attr = attr[:-1]
957	value = None
958	if not is_name(attr):
959	message.warning('%s: illegal attribute name' % syntax)
960	else:
961	if attrs is not None:
962	attrs[attr] = value
963	if name != 'set2': # set2 only updates local attributes.
964	document.attributes[attr] = value
965	if value is None:
966	result = None
967	else:
968	result = ''
969	elif name == 'include':
970	if not os.path.exists(args):
971	message.warning('%s: file does not exist' % syntax)
972	elif not is_safe_file(args):
973	message.unsafe(syntax)
974	else:
975	f = open(args)
976	try:
977	result = [s.rstrip() for s in f]
978	finally:
979	f.close()
980	if result:
981	result = subs_attrs(result)
982	result = separator.join(result)
983	result = result.expandtabs(reader.tabsize)
984	else:
985	result = ''
986	elif name == 'include1':
987	result = separator.join(config.include1[args])
988	elif name == 'template':
989	if not args in config.sections:
990	message.warning('%s: template does not exist' % syntax)
991	else:
992	result = []
993	for line in config.sections[args]:
994	line = subs_attrs(line)
995	if line is not None:
996	result.append(line)
997	result = '\n'.join(result)
998	else:
999	assert False
1000	if result and name in ('eval3','sys3'):
1001	macros.passthroughs.append(result)
1002	result = '\x07' + str(len(macros.passthroughs)-1) + '\x07'
1003	return result
1004
1005	def subs_attrs(lines, dictionary=None):
1006	"""Substitute 'lines' of text with attributes from the global
1007	document.attributes dictionary and from 'dictionary' ('dictionary'
1008	entries take precedence). Return a tuple of the substituted lines. 'lines'
1009	containing undefined attributes are deleted. If 'lines' is a string then
1010	return a string.
1011
1012	- Attribute references are substituted in the following order: simple,
1013	conditional, system.
1014	- Attribute references inside 'dictionary' entry values are substituted.
1015	"""
1016
1017	def end_brace(text,start):
1018	"""Return index following end brace that matches brace at start in
1019	text."""
1020	assert text[start] == '{'
1021	n = 0
1022	result = start
1023	for c in text[start:]:
1024	# Skip braces that are followed by a backslash.
1025	if result == len(text)-1 or text[result+1] != '\\':
1026	if c == '{': n = n + 1
1027	elif c == '}': n = n - 1
1028	result = result + 1
1029	if n == 0: break
1030	return result
1031
1032	if type(lines) == str:
1033	string_result = True
1034	lines = [lines]
1035	else:
1036	string_result = False
1037	if dictionary is None:
1038	attrs = document.attributes
1039	else:
1040	# Remove numbered document attributes so they don't clash with
1041	# attribute list positional attributes.
1042	attrs = {}
1043	for k,v in document.attributes.items():
1044	if not re.match(r'^\d+$', k):
1045	attrs[k] = v
1046	# Substitute attribute references inside dictionary values.
1047	for k,v in dictionary.items():
1048	if v is None:
1049	del dictionary[k]
1050	else:
1051	v = subs_attrs(str(v))
1052	if v is None:
1053	del dictionary[k]
1054	else:
1055	dictionary[k] = v
1056	attrs.update(dictionary)
1057	# Substitute all attributes in all lines.
1058	result = []
1059	for line in lines:
1060	# Make it easier for regular expressions.
1061	line = line.replace('\\{','{\\')
1062	line = line.replace('\\}','}\\')
1063	# Expand simple attributes ({name}).
1064	# Nested attributes not allowed.
1065	reo = re.compile(r'(?su)\{(?P<name>[^\\\W][-\w]*?)\}(?!\\)')
1066	pos = 0
1067	while True:
1068	mo = reo.search(line,pos)
1069	if not mo: break
1070	s = attrs.get(mo.group('name'))
1071	if s is None:
1072	pos = mo.end()
1073	else:
1074	s = str(s)
1075	line = line[:mo.start()] + s + line[mo.end():]
1076	pos = mo.start() + len(s)
1077	# Expand conditional attributes.
1078	# Single name -- higher precedence.
1079	reo1 = re.compile(r'(?su)\{(?P<name>[^\\\W][-\w]*?)' \
1080	r'(?P<op>\=\|\?\|!\|#\|%\|@\|\$)' \
1081	r'(?P<value>.*?)\}(?!\\)')
1082	# Multiple names (n1,n2,... or n1+n2+...) -- lower precedence.
1083	reo2 = re.compile(r'(?su)\{(?P<name>[^\\\W][-\w'+OR+AND+r']*?)' \
1084	r'(?P<op>\=\|\?\|!\|#\|%\|@\|\$)' \
1085	r'(?P<value>.*?)\}(?!\\)')
1086	for reo in [reo1,reo2]:
1087	pos = 0
1088	while True:
1089	mo = reo.search(line,pos)
1090	if not mo: break
1091	attr = mo.group()
1092	name = mo.group('name')
1093	if reo == reo2:
1094	if OR in name:
1095	sep = OR
1096	else:
1097	sep = AND
1098	names = [s.strip() for s in name.split(sep) if s.strip() ]
1099	for n in names:
1100	if not re.match(r'^[^\\\W][-\w]*$',n):
1101	message.error('illegal attribute syntax: %s' % attr)
1102	if sep == OR:
1103	# Process OR name expression: n1,n2,...
1104	for n in names:
1105	if attrs.get(n) is not None:
1106	lval = ''
1107	break
1108	else:
1109	lval = None
1110	else:
1111	# Process AND name expression: n1+n2+...
1112	for n in names:
1113	if attrs.get(n) is None:
1114	lval = None
1115	break
1116	else:
1117	lval = ''
1118	else:
1119	lval = attrs.get(name)
1120	op = mo.group('op')
1121	# mo.end() not good enough because '{x={y}}' matches '{x={y}'.
1122	end = end_brace(line,mo.start())
1123	rval = line[mo.start('value'):end-1]
1124	UNDEFINED = '{zzzzz}'
1125	if lval is None:
1126	if op == '=': s = rval
1127	elif op == '?': s = ''
1128	elif op == '!': s = rval
1129	elif op == '#': s = UNDEFINED # So the line is dropped.
1130	elif op == '%': s = rval
1131	elif op in ('@','$'):
1132	s = UNDEFINED # So the line is dropped.
1133	else:
1134	assert False, 'illegal attribute: %s' % attr
1135	else:
1136	if op == '=': s = lval
1137	elif op == '?': s = rval
1138	elif op == '!': s = ''
1139	elif op == '#': s = rval
1140	elif op == '%': s = UNDEFINED # So the line is dropped.
1141	elif op in ('@','$'):
1142	v = re.split(r'(?<!\\):',rval)
1143	if len(v) not in (2,3):
1144	message.error('illegal attribute syntax: %s' % attr)
1145	s = ''
1146	elif not is_re('^'+v[0]+'$'):
1147	message.error('illegal attribute regexp: %s' % attr)
1148	s = ''
1149	else:
1150	v = [s.replace('\\:',':') for s in v]
1151	re_mo = re.match('^'+v[0]+'$',lval)
1152	if op == '@':
1153	if re_mo:
1154	s = v[1] # {<name>@<re>:<v1>[:<v2>]}
1155	else:
1156	if len(v) == 3: # {<name>@<re>:<v1>:<v2>}
1157	s = v[2]
1158	else: # {<name>@<re>:<v1>}
1159	s = ''
1160	else:
1161	if re_mo:
1162	if len(v) == 2: # {<name>$<re>:<v1>}
1163	s = v[1]
1164	elif v[1] == '': # {<name>$<re>::<v2>}
1165	s = UNDEFINED # So the line is dropped.
1166	else: # {<name>$<re>:<v1>:<v2>}
1167	s = v[1]
1168	else:
1169	if len(v) == 2: # {<name>$<re>:<v1>}
1170	s = UNDEFINED # So the line is dropped.
1171	else: # {<name>$<re>:<v1>:<v2>}
1172	s = v[2]
1173	else:
1174	assert False, 'illegal attribute: %s' % attr
1175	s = str(s)
1176	line = line[:mo.start()] + s + line[end:]
1177	pos = mo.start() + len(s)
1178	# Drop line if it contains unsubstituted {name} references.
1179	skipped = re.search(r'(?su)\{[^\\\W][-\w]*?\}(?!\\)', line)
1180	if skipped:
1181	trace('dropped line', line)
1182	continue;
1183	# Expand system attributes (eval has precedence).
1184	reos = [
1185	re.compile(r'(?su)\{(?P<action>eval):(?P<expr>.*?)\}(?!\\)'),
1186	re.compile(r'(?su)\{(?P<action>[^\\\W][-\w]?):(?P<expr>.?)\}(?!\\)'),
1187	]
1188	skipped = False
1189	for reo in reos:
1190	pos = 0
1191	while True:
1192	mo = reo.search(line,pos)
1193	if not mo: break
1194	expr = mo.group('expr')
1195	action = mo.group('action')
1196	expr = expr.replace('{\\','{')
1197	expr = expr.replace('}\\','}')
1198	s = system(action, expr, attrs=dictionary)
1199	if dictionary is not None and action in ('counter','counter2','set','set2'):
1200	# These actions create and update attributes.
1201	attrs.update(dictionary)
1202	if s is None:
1203	# Drop line if the action returns None.
1204	skipped = True
1205	break
1206	line = line[:mo.start()] + s + line[mo.end():]
1207	pos = mo.start() + len(s)
1208	if skipped:
1209	break
1210	if not skipped:
1211	# Remove backslash from escaped entries.
1212	line = line.replace('{\\','{')
1213	line = line.replace('}\\','}')
1214	result.append(line)
1215	if string_result:
1216	if result:
1217	return '\n'.join(result)
1218	else:
1219	return None
1220	else:
1221	return tuple(result)
1222
1223	def char_encoding():
1224	encoding = document.attributes.get('encoding')
1225	if encoding:
1226	try:
1227	codecs.lookup(encoding)
1228	except LookupError,e:
1229	raise EAsciiDoc,str(e)
1230	return encoding
1231
1232	def char_len(s):
1233	return len(char_decode(s))
1234
1235	east_asian_widths = {'W': 2, # Wide
1236	'F': 2, # Full-width (wide)
1237	'Na': 1, # Narrow
1238	'H': 1, # Half-width (narrow)
1239	'N': 1, # Neutral (not East Asian, treated as narrow)
1240	'A': 1} # Ambiguous (s/b wide in East Asian context,
1241	# narrow otherwise, but that doesn't work)
1242	"""Mapping of result codes from `unicodedata.east_asian_width()` to character
1243	column widths."""
1244
1245	def column_width(s):
1246	text = char_decode(s)
1247	if isinstance(text, unicode):
1248	width = 0
1249	for c in text:
1250	width += east_asian_widths[unicodedata.east_asian_width(c)]
1251	return width
1252	else:
1253	return len(text)
1254
1255	def char_decode(s):
1256	if char_encoding():
1257	try:
1258	return s.decode(char_encoding())
1259	except Exception:
1260	raise EAsciiDoc, \
1261	"'%s' codec can't decode \"%s\"" % (char_encoding(), s)
1262	else:
1263	return s
1264
1265	def char_encode(s):
1266	if char_encoding():
1267	return s.encode(char_encoding())
1268	else:
1269	return s
1270
1271	def time_str(t):
1272	"""Convert seconds since the Epoch to formatted local time string."""
1273	t = time.localtime(t)
1274	s = time.strftime('%H:%M:%S',t)
1275	if time.daylight and t.tm_isdst == 1:
1276	result = s + ' ' + time.tzname[1]
1277	else:
1278	result = s + ' ' + time.tzname[0]
1279	# Attempt to convert the localtime to the output encoding.
1280	try:
1281	result = char_encode(result.decode(locale.getdefaultlocale()[1]))
1282	except Exception:
1283	pass
1284	return result
1285
1286	def date_str(t):
1287	"""Convert seconds since the Epoch to formatted local date string."""
1288	t = time.localtime(t)
1289	return time.strftime('%Y-%m-%d',t)
1290
1291
1292	class Lex:
1293	"""Lexical analysis routines. Static methods and attributes only."""
1294	prev_element = None
1295	prev_cursor = None
1296	def __init__(self):
1297	raise AssertionError,'no class instances allowed'
1298	@staticmethod
1299	def next():
1300	"""Returns class of next element on the input (None if EOF). The
1301	reader is assumed to be at the first line following a previous element,
1302	end of file or line one. Exits with the reader pointing to the first
1303	line of the next element or EOF (leading blank lines are skipped)."""
1304	reader.skip_blank_lines()
1305	if reader.eof(): return None
1306	# Optimization: If we've already checked for an element at this
1307	# position return the element.
1308	if Lex.prev_element and Lex.prev_cursor == reader.cursor:
1309	return Lex.prev_element
1310	if AttributeEntry.isnext():
1311	result = AttributeEntry
1312	elif AttributeList.isnext():
1313	result = AttributeList
1314	elif BlockTitle.isnext() and not tables_OLD.isnext():
1315	result = BlockTitle
1316	elif Title.isnext():
1317	if AttributeList.style() == 'float':
1318	result = FloatingTitle
1319	else:
1320	result = Title
1321	elif macros.isnext():
1322	result = macros.current
1323	elif lists.isnext():
1324	result = lists.current
1325	elif blocks.isnext():
1326	result = blocks.current
1327	elif tables_OLD.isnext():
1328	result = tables_OLD.current
1329	elif tables.isnext():
1330	result = tables.current
1331	else:
1332	if not paragraphs.isnext():
1333	raise EAsciiDoc,'paragraph expected'
1334	result = paragraphs.current
1335	# Optimization: Cache answer.
1336	Lex.prev_cursor = reader.cursor
1337	Lex.prev_element = result
1338	return result
1339
1340	@staticmethod
1341	def canonical_subs(options):
1342	"""Translate composite subs values."""
1343	if len(options) == 1:
1344	if options[0] == 'none':
1345	options = ()
1346	elif options[0] == 'normal':
1347	options = config.subsnormal
1348	elif options[0] == 'verbatim':
1349	options = config.subsverbatim
1350	return options
1351
1352	@staticmethod
1353	def subs_1(s,options):
1354	"""Perform substitution specified in 'options' (in 'options' order)."""
1355	if not s:
1356	return s
1357	if document.attributes.get('plaintext') is not None:
1358	options = ('specialcharacters',)
1359	result = s
1360	options = Lex.canonical_subs(options)
1361	for o in options:
1362	if o == 'specialcharacters':
1363	result = config.subs_specialchars(result)
1364	elif o == 'attributes':
1365	result = subs_attrs(result)
1366	elif o == 'quotes':
1367	result = subs_quotes(result)
1368	elif o == 'specialwords':
1369	result = config.subs_specialwords(result)
1370	elif o in ('replacements','replacements2','replacements3'):
1371	result = config.subs_replacements(result,o)
1372	elif o == 'macros':
1373	result = macros.subs(result)
1374	elif o == 'callouts':
1375	result = macros.subs(result,callouts=True)
1376	else:
1377	raise EAsciiDoc,'illegal substitution option: %s' % o
1378	trace(o, s, result)
1379	if not result:
1380	break
1381	return result
1382
1383	@staticmethod
1384	def subs(lines,options):
1385	"""Perform inline processing specified by 'options' (in 'options'
1386	order) on sequence of 'lines'."""
1387	if not lines or not options:
1388	return lines
1389	options = Lex.canonical_subs(options)
1390	# Join lines so quoting can span multiple lines.
1391	para = '\n'.join(lines)
1392	if 'macros' in options:
1393	para = macros.extract_passthroughs(para)
1394	for o in options:
1395	if o == 'attributes':
1396	# If we don't substitute attributes line-by-line then a single
1397	# undefined attribute will drop the entire paragraph.
1398	lines = subs_attrs(para.split('\n'))
1399	para = '\n'.join(lines)
1400	else:
1401	para = Lex.subs_1(para,(o,))
1402	if 'macros' in options:
1403	para = macros.restore_passthroughs(para)
1404	return para.splitlines()
1405
1406	@staticmethod
1407	def set_margin(lines, margin=0):
1408	"""Utility routine that sets the left margin to 'margin' space in a
1409	block of non-blank lines."""
1410	# Calculate width of block margin.
1411	lines = list(lines)
1412	width = len(lines[0])
1413	for s in lines:
1414	i = re.search(r'\S',s).start()
1415	if i < width: width = i
1416	# Strip margin width from all lines.
1417	for i in range(len(lines)):
1418	lines[i] = ' '*margin + lines[i][width:]
1419	return lines
1420
1421	#---------------------------------------------------------------------------
1422	# Document element classes parse AsciiDoc reader input and write DocBook writer
1423	# output.
1424	#---------------------------------------------------------------------------
1425	class Document(object):
1426
1427	# doctype property.
1428	def getdoctype(self):
1429	return self.attributes.get('doctype')
1430	def setdoctype(self,doctype):
1431	self.attributes['doctype'] = doctype
1432	doctype = property(getdoctype,setdoctype)
1433
1434	# backend property.
1435	def getbackend(self):
1436	return self.attributes.get('backend')
1437	def setbackend(self,backend):
1438	if backend:
1439	backend = self.attributes.get('backend-alias-' + backend, backend)
1440	self.attributes['backend'] = backend
1441	backend = property(getbackend,setbackend)
1442
1443	def __init__(self):
1444	self.infile = None # Source file name.
1445	self.outfile = None # Output file name.
1446	self.attributes = InsensitiveDict()
1447	self.level = 0 # 0 => front matter. 1,2,3 => sect1,2,3.
1448	self.has_errors = False # Set true if processing errors were flagged.
1449	self.has_warnings = False # Set true if warnings were flagged.
1450	self.safe = False # Default safe mode.
1451	def update_attributes(self,attrs=None):
1452	"""
1453	Set implicit attributes and attributes in 'attrs'.
1454	"""
1455	t = time.time()
1456	self.attributes['localtime'] = time_str(t)
1457	self.attributes['localdate'] = date_str(t)
1458	self.attributes['asciidoc-version'] = VERSION
1459	self.attributes['asciidoc-file'] = APP_FILE
1460	self.attributes['asciidoc-dir'] = APP_DIR
1461	if localapp():
1462	self.attributes['asciidoc-confdir'] = APP_DIR
1463	else:
1464	self.attributes['asciidoc-confdir'] = CONF_DIR
1465	self.attributes['user-dir'] = USER_DIR
1466	if config.verbose:
1467	self.attributes['verbose'] = ''
1468	# Update with configuration file attributes.
1469	if attrs:
1470	self.attributes.update(attrs)
1471	# Update with command-line attributes.
1472	self.attributes.update(config.cmd_attrs)
1473	# Extract miscellaneous configuration section entries from attributes.
1474	if attrs:
1475	config.load_miscellaneous(attrs)
1476	config.load_miscellaneous(config.cmd_attrs)
1477	self.attributes['newline'] = config.newline
1478	# File name related attributes can't be overridden.
1479	if self.infile is not None:
1480	if self.infile and os.path.exists(self.infile):
1481	t = os.path.getmtime(self.infile)
1482	elif self.infile == '<stdin>':
1483	t = time.time()
1484	else:
1485	t = None
1486	if t:
1487	self.attributes['doctime'] = time_str(t)
1488	self.attributes['docdate'] = date_str(t)
1489	if self.infile != '<stdin>':
1490	self.attributes['infile'] = self.infile
1491	self.attributes['indir'] = os.path.dirname(self.infile)
1492	self.attributes['docfile'] = self.infile
1493	self.attributes['docdir'] = os.path.dirname(self.infile)
1494	self.attributes['docname'] = os.path.splitext(
1495	os.path.basename(self.infile))[0]
1496	if self.outfile:
1497	if self.outfile != '<stdout>':
1498	self.attributes['outfile'] = self.outfile
1499	self.attributes['outdir'] = os.path.dirname(self.outfile)
1500	if self.infile == '<stdin>':
1501	self.attributes['docname'] = os.path.splitext(
1502	os.path.basename(self.outfile))[0]
1503	ext = os.path.splitext(self.outfile)[1][1:]
1504	elif config.outfilesuffix:
1505	ext = config.outfilesuffix[1:]
1506	else:
1507	ext = ''
1508	if ext:
1509	self.attributes['filetype'] = ext
1510	self.attributes['filetype-'+ext] = ''
1511	def load_lang(self):
1512	"""
1513	Load language configuration file.
1514	"""
1515	lang = self.attributes.get('lang')
1516	if lang is None:
1517	filename = 'lang-en.conf' # Default language file.
1518	else:
1519	filename = 'lang-' + lang + '.conf'
1520	if config.load_from_dirs(filename):
1521	self.attributes['lang'] = lang # Reinstate new lang attribute.
1522	else:
1523	if lang is None:
1524	# The default language file must exist.
1525	message.error('missing conf file: %s' % filename, halt=True)
1526	else:
1527	message.warning('missing language conf file: %s' % filename)
1528	def set_deprecated_attribute(self,old,new):
1529	"""
1530	Ensures the 'old' name of an attribute that was renamed to 'new' is
1531	still honored.
1532	"""
1533	if self.attributes.get(new) is None:
1534	if self.attributes.get(old) is not None:
1535	self.attributes[new] = self.attributes[old]
1536	else:
1537	self.attributes[old] = self.attributes[new]
1538	def consume_attributes_and_comments(self,comments_only=False,noblanks=False):
1539	"""
1540	Returns True if one or more attributes or comments were consumed.
1541	If 'noblanks' is True then consumation halts if a blank line is
1542	encountered.
1543	"""
1544	result = False
1545	finished = False
1546	while not finished:
1547	finished = True
1548	if noblanks and not reader.read_next(): return result
1549	if blocks.isnext() and 'skip' in blocks.current.options:
1550	result = True
1551	finished = False
1552	blocks.current.translate()
1553	if noblanks and not reader.read_next(): return result
1554	if macros.isnext() and macros.current.name == 'comment':
1555	result = True
1556	finished = False
1557	macros.current.translate()
1558	if not comments_only:
1559	if AttributeEntry.isnext():
1560	result = True
1561	finished = False
1562	AttributeEntry.translate()
1563	if AttributeList.isnext():
1564	result = True
1565	finished = False
1566	AttributeList.translate()
1567	return result
1568	def parse_header(self,doctype,backend):
1569	"""
1570	Parses header, sets corresponding document attributes and finalizes
1571	document doctype and backend properties.
1572	Returns False if the document does not have a header.
1573	'doctype' and 'backend' are the doctype and backend option values
1574	passed on the command-line, None if no command-line option was not
1575	specified.
1576	"""
1577	assert self.level == 0
1578	# Skip comments and attribute entries that preceed the header.
1579	self.consume_attributes_and_comments()
1580	if doctype is not None:
1581	# Command-line overrides header.
1582	self.doctype = doctype
1583	elif self.doctype is None:
1584	# Was not set on command-line or in document header.
1585	self.doctype = DEFAULT_DOCTYPE
1586	# Process document header.
1587	has_header = (Title.isnext() and Title.level == 0
1588	and AttributeList.style() != 'float')
1589	if self.doctype == 'manpage' and not has_header:
1590	message.error('manpage document title is mandatory',halt=True)
1591	if has_header:
1592	Header.parse()
1593	# Command-line entries override header derived entries.
1594	self.attributes.update(config.cmd_attrs)
1595	# DEPRECATED: revision renamed to revnumber.
1596	self.set_deprecated_attribute('revision','revnumber')
1597	# DEPRECATED: date renamed to revdate.
1598	self.set_deprecated_attribute('date','revdate')
1599	if doctype is not None:
1600	# Command-line overrides header.
1601	self.doctype = doctype
1602	if backend is not None:
1603	# Command-line overrides header.
1604	self.backend = backend
1605	elif self.backend is None:
1606	# Was not set on command-line or in document header.
1607	self.backend = DEFAULT_BACKEND
1608	else:
1609	# Has been set in document header.
1610	self.backend = self.backend # Translate alias in header.
1611	assert self.doctype in ('article','manpage','book'), 'illegal document type'
1612	return has_header
1613	def translate(self,has_header):
1614	if self.doctype == 'manpage':
1615	# Translate mandatory NAME section.
1616	if Lex.next() is not Title:
1617	message.error('name section expected')
1618	else:
1619	Title.translate()
1620	if Title.level != 1:
1621	message.error('name section title must be at level 1')
1622	if not isinstance(Lex.next(),Paragraph):
1623	message.error('malformed name section body')
1624	lines = reader.read_until(r'^$')
1625	s = ' '.join(lines)
1626	mo = re.match(r'^(?P<manname>.?)\s+-\s+(?P<manpurpose>.)$',s)
1627	if not mo:
1628	message.error('malformed name section body')
1629	self.attributes['manname'] = mo.group('manname').strip()
1630	self.attributes['manpurpose'] = mo.group('manpurpose').strip()
1631	names = [s.strip() for s in self.attributes['manname'].split(',')]
1632	if len(names) > 9:
1633	message.warning('too many manpage names')
1634	for i,name in enumerate(names):
1635	self.attributes['manname%d' % (i+1)] = name
1636	if has_header:
1637	# Do postponed substitutions (backend confs have been loaded).
1638	self.attributes['doctitle'] = Title.dosubs(self.attributes['doctitle'])
1639	if config.header_footer:
1640	hdr = config.subs_section('header',{})
1641	writer.write(hdr,trace='header')
1642	if 'title' in self.attributes:
1643	del self.attributes['title']
1644	self.consume_attributes_and_comments()
1645	if self.doctype in ('article','book'):
1646	# Translate 'preamble' (untitled elements between header
1647	# and first section title).
1648	if Lex.next() is not Title:
1649	stag,etag = config.section2tags('preamble')
1650	writer.write(stag,trace='preamble open')
1651	Section.translate_body()
1652	writer.write(etag,trace='preamble close')
1653	elif self.doctype == 'manpage' and 'name' in config.sections:
1654	writer.write(config.subs_section('name',{}), trace='name')
1655	else:
1656	self.process_author_names()
1657	if config.header_footer:
1658	hdr = config.subs_section('header',{})
1659	writer.write(hdr,trace='header')
1660	if Lex.next() is not Title:
1661	Section.translate_body()
1662	# Process remaining sections.
1663	while not reader.eof():
1664	if Lex.next() is not Title:
1665	raise EAsciiDoc,'section title expected'
1666	Section.translate()
1667	Section.setlevel(0) # Write remaining unwritten section close tags.
1668	# Substitute document parameters and write document footer.
1669	if config.header_footer:
1670	ftr = config.subs_section('footer',{})
1671	writer.write(ftr,trace='footer')
1672	def parse_author(self,s):
1673	""" Return False if the author is malformed."""
1674	attrs = self.attributes # Alias for readability.
1675	s = s.strip()
1676	mo = re.match(r'^(?P<name1>[^<>\s]+)'
1677	'(\s+(?P<name2>[^<>\s]+))?'
1678	'(\s+(?P<name3>[^<>\s]+))?'
1679	'(\s+<(?P<email>\S+)>)?$',s)
1680	if not mo:
1681	# Names that don't match the formal specification.
1682	if s:
1683	attrs['firstname'] = s
1684	return
1685	firstname = mo.group('name1')
1686	if mo.group('name3'):
1687	middlename = mo.group('name2')
1688	lastname = mo.group('name3')
1689	else:
1690	middlename = None
1691	lastname = mo.group('name2')
1692	firstname = firstname.replace('_',' ')
1693	if middlename:
1694	middlename = middlename.replace('_',' ')
1695	if lastname:
1696	lastname = lastname.replace('_',' ')
1697	email = mo.group('email')
1698	if firstname:
1699	attrs['firstname'] = firstname
1700	if middlename:
1701	attrs['middlename'] = middlename
1702	if lastname:
1703	attrs['lastname'] = lastname
1704	if email:
1705	attrs['email'] = email
1706	return
1707	def process_author_names(self):
1708	""" Calculate any missing author related attributes."""
1709	attrs = self.attributes # Alias for readability.
1710	firstname = attrs.get('firstname','')
1711	middlename = attrs.get('middlename','')
1712	lastname = attrs.get('lastname','')
1713	author = attrs.get('author')
1714	initials = attrs.get('authorinitials')
1715	if author and not (firstname or middlename or lastname):
1716	self.parse_author(author)
1717	attrs['author'] = author.replace('_',' ')
1718	self.process_author_names()
1719	return
1720	if not author:
1721	author = '%s %s %s' % (firstname, middlename, lastname)
1722	author = author.strip()
1723	author = re.sub(r'\s+',' ', author)
1724	if not initials:
1725	initials = (char_decode(firstname)[:1] +
1726	char_decode(middlename)[:1] + char_decode(lastname)[:1])
1727	initials = char_encode(initials).upper()
1728	names = [firstname,middlename,lastname,author,initials]
1729	for i,v in enumerate(names):
1730	v = config.subs_specialchars(v)
1731	v = subs_attrs(v)
1732	names[i] = v
1733	firstname,middlename,lastname,author,initials = names
1734	if firstname:
1735	attrs['firstname'] = firstname
1736	if middlename:
1737	attrs['middlename'] = middlename
1738	if lastname:
1739	attrs['lastname'] = lastname
1740	if author:
1741	attrs['author'] = author
1742	if initials:
1743	attrs['authorinitials'] = initials
1744	if author:
1745	attrs['authored'] = ''
1746
1747
1748	class Header:
1749	"""Static methods and attributes only."""
1750	REV_LINE_RE = r'^(\D(?P<revnumber>.?),)?(?P<revdate>.?)(:\s(?P<revremark>.*))?$'
1751	RCS_ID_RE = r'^\$Id: \S+ (?P<revnumber>\S+) (?P<revdate>\S+) \S+ (?P<author>\S+) (\S+ )?\$$'
1752	def __init__(self):
1753	raise AssertionError,'no class instances allowed'
1754	@staticmethod
1755	def parse():
1756	assert Lex.next() is Title and Title.level == 0
1757	attrs = document.attributes # Alias for readability.
1758	# Postpone title subs until backend conf files have been loaded.
1759	Title.translate(skipsubs=True)
1760	attrs['doctitle'] = Title.attributes['title']
1761	document.consume_attributes_and_comments(noblanks=True)
1762	s = reader.read_next()
1763	mo = None
1764	if s:
1765	# Process first header line after the title that is not a comment
1766	# or an attribute entry.
1767	s = reader.read()
1768	mo = re.match(Header.RCS_ID_RE,s)
1769	if not mo:
1770	document.parse_author(s)
1771	document.consume_attributes_and_comments(noblanks=True)
1772	if reader.read_next():
1773	# Process second header line after the title that is not a
1774	# comment or an attribute entry.
1775	s = reader.read()
1776	s = subs_attrs(s)
1777	if s:
1778	mo = re.match(Header.RCS_ID_RE,s)
1779	if not mo:
1780	mo = re.match(Header.REV_LINE_RE,s)
1781	document.consume_attributes_and_comments(noblanks=True)
1782	s = attrs.get('revnumber')
1783	if s:
1784	mo = re.match(Header.RCS_ID_RE,s)
1785	if mo:
1786	revnumber = mo.group('revnumber')
1787	if revnumber:
1788	attrs['revnumber'] = revnumber.strip()
1789	author = mo.groupdict().get('author')
1790	if author and 'firstname' not in attrs:
1791	document.parse_author(author)
1792	revremark = mo.groupdict().get('revremark')
1793	if revremark is not None:
1794	revremark = [revremark]
1795	# Revision remarks can continue on following lines.
1796	while reader.read_next():
1797	if document.consume_attributes_and_comments(noblanks=True):
1798	break
1799	revremark.append(reader.read())
1800	revremark = Lex.subs(revremark,['normal'])
1801	revremark = '\n'.join(revremark).strip()
1802	attrs['revremark'] = revremark
1803	revdate = mo.group('revdate')
1804	if revdate:
1805	attrs['revdate'] = revdate.strip()
1806	elif revnumber or revremark:
1807	# Set revision date to ensure valid DocBook revision.
1808	attrs['revdate'] = attrs['docdate']
1809	document.process_author_names()
1810	if document.doctype == 'manpage':
1811	# manpage title formatted like mantitle(manvolnum).
1812	mo = re.match(r'^(?P<mantitle>.)\((?P<manvolnum>.)\)$',
1813	attrs['doctitle'])
1814	if not mo:
1815	message.error('malformed manpage title')
1816	else:
1817	mantitle = mo.group('mantitle').strip()
1818	mantitle = subs_attrs(mantitle)
1819	if mantitle is None:
1820	message.error('undefined attribute in manpage title')
1821	# mantitle is lowered only if in ALL CAPS
1822	if mantitle == mantitle.upper():
1823	mantitle = mantitle.lower()
1824	attrs['mantitle'] = mantitle;
1825	attrs['manvolnum'] = mo.group('manvolnum').strip()
1826
1827	class AttributeEntry:
1828	"""Static methods and attributes only."""
1829	pattern = None
1830	subs = None
1831	name = None
1832	name2 = None
1833	value = None
1834	attributes = {} # Accumulates all the parsed attribute entries.
1835	def __init__(self):
1836	raise AssertionError,'no class instances allowed'
1837	@staticmethod
1838	def isnext():
1839	result = False # Assume not next.
1840	if not AttributeEntry.pattern:
1841	pat = document.attributes.get('attributeentry-pattern')
1842	if not pat:
1843	message.error("[attributes] missing 'attributeentry-pattern' entry")
1844	AttributeEntry.pattern = pat
1845	line = reader.read_next()
1846	if line:
1847	# Attribute entry formatted like :<name>[.<name2>]:[ <value>]
1848	mo = re.match(AttributeEntry.pattern,line)
1849	if mo:
1850	AttributeEntry.name = mo.group('attrname')
1851	AttributeEntry.name2 = mo.group('attrname2')
1852	AttributeEntry.value = mo.group('attrvalue') or ''
1853	AttributeEntry.value = AttributeEntry.value.strip()
1854	result = True
1855	return result
1856	@staticmethod
1857	def translate():
1858	assert Lex.next() is AttributeEntry
1859	attr = AttributeEntry # Alias for brevity.
1860	reader.read() # Discard attribute entry from reader.
1861	while attr.value.endswith(' +'):
1862	if not reader.read_next(): break
1863	attr.value = attr.value[:-1] + reader.read().strip()
1864	if attr.name2 is not None:
1865	# Configuration file attribute.
1866	if attr.name2 != '':
1867	# Section entry attribute.
1868	section = {}
1869	# Some sections can have name! syntax.
1870	if attr.name in ('attributes','miscellaneous') and attr.name2[-1] == '!':
1871	section[attr.name] = [attr.name2]
1872	else:
1873	section[attr.name] = ['%s=%s' % (attr.name2,attr.value)]
1874	config.load_sections(section)
1875	config.load_miscellaneous(config.conf_attrs)
1876	else:
1877	# Markup template section attribute.
1878	config.sections[attr.name] = [attr.value]
1879	else:
1880	# Normal attribute.
1881	if attr.name[-1] == '!':
1882	# Names like name! undefine the attribute.
1883	attr.name = attr.name[:-1]
1884	attr.value = None
1885	# Strip white space and illegal name chars.
1886	attr.name = re.sub(r'(?u)[^\w\-_]', '', attr.name).lower()
1887	# Don't override most command-line attributes.
1888	if attr.name in config.cmd_attrs \
1889	and attr.name not in ('trace','numbered'):
1890	return
1891	# Update document attributes with attribute value.
1892	if attr.value is not None:
1893	mo = re.match(r'^pass:(?P<attrs>.)\[(?P<value>.)\]$', attr.value)
1894	if mo:
1895	# Inline passthrough syntax.
1896	attr.subs = mo.group('attrs')
1897	attr.value = mo.group('value') # Passthrough.
1898	else:
1899	# Default substitution.
1900	# DEPRECATED: attributeentry-subs
1901	attr.subs = document.attributes.get('attributeentry-subs',
1902	'specialcharacters,attributes')
1903	attr.subs = parse_options(attr.subs, SUBS_OPTIONS,
1904	'illegal substitution option')
1905	attr.value = Lex.subs((attr.value,), attr.subs)
1906	attr.value = writer.newline.join(attr.value)
1907	document.attributes[attr.name] = attr.value
1908	elif attr.name in document.attributes:
1909	del document.attributes[attr.name]
1910	attr.attributes[attr.name] = attr.value
1911
1912	class AttributeList:
1913	"""Static methods and attributes only."""
1914	pattern = None
1915	match = None
1916	attrs = {}
1917	def __init__(self):
1918	raise AssertionError,'no class instances allowed'
1919	@staticmethod
1920	def initialize():
1921	if not 'attributelist-pattern' in document.attributes:
1922	message.error("[attributes] missing 'attributelist-pattern' entry")
1923	AttributeList.pattern = document.attributes['attributelist-pattern']
1924	@staticmethod
1925	def isnext():
1926	result = False # Assume not next.
1927	line = reader.read_next()
1928	if line:
1929	mo = re.match(AttributeList.pattern, line)
1930	if mo:
1931	AttributeList.match = mo
1932	result = True
1933	return result
1934	@staticmethod
1935	def translate():
1936	assert Lex.next() is AttributeList
1937	reader.read() # Discard attribute list from reader.
1938	attrs = {}
1939	d = AttributeList.match.groupdict()
1940	for k,v in d.items():
1941	if v is not None:
1942	if k == 'attrlist':
1943	v = subs_attrs(v)
1944	if v:
1945	parse_attributes(v, attrs)
1946	else:
1947	AttributeList.attrs[k] = v
1948	AttributeList.subs(attrs)
1949	AttributeList.attrs.update(attrs)
1950	@staticmethod
1951	def subs(attrs):
1952	'''Substitute single quoted attribute values normally.'''
1953	reo = re.compile(r"^'.*'$")
1954	for k,v in attrs.items():
1955	if reo.match(str(v)):
1956	attrs[k] = Lex.subs_1(v[1:-1], config.subsnormal)
1957	@staticmethod
1958	def style():
1959	return AttributeList.attrs.get('style') or AttributeList.attrs.get('1')
1960	@staticmethod
1961	def consume(d={}):
1962	"""Add attribute list to the dictionary 'd' and reset the list."""
1963	if AttributeList.attrs:
1964	d.update(AttributeList.attrs)
1965	AttributeList.attrs = {}
1966	# Generate option attributes.
1967	if 'options' in d:
1968	options = parse_options(d['options'], (), 'illegal option name')
1969	for option in options:
1970	d[option+'-option'] = ''
1971
1972	class BlockTitle:
1973	"""Static methods and attributes only."""
1974	title = None
1975	pattern = None
1976	def __init__(self):
1977	raise AssertionError,'no class instances allowed'
1978	@staticmethod
1979	def isnext():
1980	result = False # Assume not next.
1981	line = reader.read_next()
1982	if line:
1983	mo = re.match(BlockTitle.pattern,line)
1984	if mo:
1985	BlockTitle.title = mo.group('title')
1986	result = True
1987	return result
1988	@staticmethod
1989	def translate():
1990	assert Lex.next() is BlockTitle
1991	reader.read() # Discard title from reader.
1992	# Perform title substitutions.
1993	if not Title.subs:
1994	Title.subs = config.subsnormal
1995	s = Lex.subs((BlockTitle.title,), Title.subs)
1996	s = writer.newline.join(s)
1997	if not s:
1998	message.warning('blank block title')
1999	BlockTitle.title = s
2000	@staticmethod
2001	def consume(d={}):
2002	"""If there is a title add it to dictionary 'd' then reset title."""
2003	if BlockTitle.title:
2004	d['title'] = BlockTitle.title
2005	BlockTitle.title = None
2006
2007	class Title:
2008	"""Processes Header and Section titles. Static methods and attributes
2009	only."""
2010	# Class variables
2011	underlines = ('==','--','~~','^^','++') # Levels 0,1,2,3,4.
2012	subs = ()
2013	pattern = None
2014	level = 0
2015	attributes = {}
2016	sectname = None
2017	section_numbers = [0]*len(underlines)
2018	dump_dict = {}
2019	linecount = None # Number of lines in title (1 or 2).
2020	def __init__(self):
2021	raise AssertionError,'no class instances allowed'
2022	@staticmethod
2023	def translate(skipsubs=False):
2024	"""Parse the Title.attributes and Title.level from the reader. The
2025	real work has already been done by parse()."""
2026	assert Lex.next() in (Title,FloatingTitle)
2027	# Discard title from reader.
2028	for i in range(Title.linecount):
2029	reader.read()
2030	Title.setsectname()
2031	if not skipsubs:
2032	Title.attributes['title'] = Title.dosubs(Title.attributes['title'])
2033	@staticmethod
2034	def dosubs(title):
2035	"""
2036	Perform title substitutions.
2037	"""
2038	if not Title.subs:
2039	Title.subs = config.subsnormal
2040	title = Lex.subs((title,), Title.subs)
2041	title = writer.newline.join(title)
2042	if not title:
2043	message.warning('blank section title')
2044	return title
2045	@staticmethod
2046	def isnext():
2047	lines = reader.read_ahead(2)
2048	return Title.parse(lines)
2049	@staticmethod
2050	def parse(lines):
2051	"""Parse title at start of lines tuple."""
2052	if len(lines) == 0: return False
2053	if len(lines[0]) == 0: return False # Title can't be blank.
2054	# Check for single-line titles.
2055	result = False
2056	for level in range(len(Title.underlines)):
2057	k = 'sect%s' % level
2058	if k in Title.dump_dict:
2059	mo = re.match(Title.dump_dict[k], lines[0])
2060	if mo:
2061	Title.attributes = mo.groupdict()
2062	Title.level = level
2063	Title.linecount = 1
2064	result = True
2065	break
2066	if not result:
2067	# Check for double-line titles.
2068	if not Title.pattern: return False # Single-line titles only.
2069	if len(lines) < 2: return False
2070	title,ul = lines[:2]
2071	title_len = column_width(title)
2072	ul_len = char_len(ul)
2073	if ul_len < 2: return False
2074	# Fast elimination check.
2075	if ul[:2] not in Title.underlines: return False
2076	# Length of underline must be within +-3 of title.
2077	if not ((ul_len-3 < title_len < ul_len+3)
2078	# Next test for backward compatibility.
2079	or (ul_len-3 < char_len(title) < ul_len+3)):
2080	return False
2081	# Check for valid repetition of underline character pairs.
2082	s = ul[:2]*((ul_len+1)/2)
2083	if ul != s[:ul_len]: return False
2084	# Don't be fooled by back-to-back delimited blocks, require at
2085	# least one alphanumeric character in title.
2086	if not re.search(r'(?u)\w',title): return False
2087	mo = re.match(Title.pattern, title)
2088	if mo:
2089	Title.attributes = mo.groupdict()
2090	Title.level = list(Title.underlines).index(ul[:2])
2091	Title.linecount = 2
2092	result = True
2093	# Check for expected pattern match groups.
2094	if result:
2095	if not 'title' in Title.attributes:
2096	message.warning('[titles] entry has no <title> group')
2097	Title.attributes['title'] = lines[0]
2098	for k,v in Title.attributes.items():
2099	if v is None: del Title.attributes[k]
2100	try:
2101	Title.level += int(document.attributes.get('leveloffset','0'))
2102	except:
2103	pass
2104	Title.attributes['level'] = str(Title.level)
2105	return result
2106	@staticmethod
2107	def load(entries):
2108	"""Load and validate [titles] section entries dictionary."""
2109	if 'underlines' in entries:
2110	errmsg = 'malformed [titles] underlines entry'
2111	try:
2112	underlines = parse_list(entries['underlines'])
2113	except Exception:
2114	raise EAsciiDoc,errmsg
2115	if len(underlines) != len(Title.underlines):
2116	raise EAsciiDoc,errmsg
2117	for s in underlines:
2118	if len(s) !=2:
2119	raise EAsciiDoc,errmsg
2120	Title.underlines = tuple(underlines)
2121	Title.dump_dict['underlines'] = entries['underlines']
2122	if 'subs' in entries:
2123	Title.subs = parse_options(entries['subs'], SUBS_OPTIONS,
2124	'illegal [titles] subs entry')
2125	Title.dump_dict['subs'] = entries['subs']
2126	if 'sectiontitle' in entries:
2127	pat = entries['sectiontitle']
2128	if not pat or not is_re(pat):
2129	raise EAsciiDoc,'malformed [titles] sectiontitle entry'
2130	Title.pattern = pat
2131	Title.dump_dict['sectiontitle'] = pat
2132	if 'blocktitle' in entries:
2133	pat = entries['blocktitle']
2134	if not pat or not is_re(pat):
2135	raise EAsciiDoc,'malformed [titles] blocktitle entry'
2136	BlockTitle.pattern = pat
2137	Title.dump_dict['blocktitle'] = pat
2138	# Load single-line title patterns.
2139	for k in ('sect0','sect1','sect2','sect3','sect4'):
2140	if k in entries:
2141	pat = entries[k]
2142	if not pat or not is_re(pat):
2143	raise EAsciiDoc,'malformed [titles] %s entry' % k
2144	Title.dump_dict[k] = pat
2145	# TODO: Check we have either a Title.pattern or at least one
2146	# single-line title pattern -- can this be done here or do we need
2147	# check routine like the other block checkers?
2148	@staticmethod
2149	def dump():
2150	dump_section('titles',Title.dump_dict)
2151	@staticmethod
2152	def setsectname():
2153	"""
2154	Set Title section name:
2155	If the first positional or 'template' attribute is set use it,
2156	next search for section title in [specialsections],
2157	if not found use default 'sect<level>' name.
2158	"""
2159	sectname = AttributeList.attrs.get('1')
2160	if sectname and sectname != 'float':
2161	Title.sectname = sectname
2162	elif 'template' in AttributeList.attrs:
2163	Title.sectname = AttributeList.attrs['template']
2164	else:
2165	for pat,sect in config.specialsections.items():
2166	mo = re.match(pat,Title.attributes['title'])
2167	if mo:
2168	title = mo.groupdict().get('title')
2169	if title is not None:
2170	Title.attributes['title'] = title.strip()
2171	else:
2172	Title.attributes['title'] = mo.group().strip()
2173	Title.sectname = sect
2174	break
2175	else:
2176	Title.sectname = 'sect%d' % Title.level
2177	@staticmethod
2178	def getnumber(level):
2179	"""Return next section number at section 'level' formatted like
2180	1.2.3.4."""
2181	number = ''
2182	for l in range(len(Title.section_numbers)):
2183	n = Title.section_numbers[l]
2184	if l == 0:
2185	continue
2186	elif l < level:
2187	number = '%s%d.' % (number, n)
2188	elif l == level:
2189	number = '%s%d.' % (number, n + 1)
2190	Title.section_numbers[l] = n + 1
2191	elif l > level:
2192	# Reset unprocessed section levels.
2193	Title.section_numbers[l] = 0
2194	return number
2195
2196
2197	class FloatingTitle(Title):
2198	'''Floated titles are translated differently.'''
2199	@staticmethod
2200	def isnext():
2201	return Title.isnext() and AttributeList.style() == 'float'
2202	@staticmethod
2203	def translate():
2204	assert Lex.next() is FloatingTitle
2205	Title.translate()
2206	Section.set_id()
2207	AttributeList.consume(Title.attributes)
2208	template = 'floatingtitle'
2209	if template in config.sections:
2210	stag,etag = config.section2tags(template,Title.attributes)
2211	writer.write(stag,trace='floating title')
2212	else:
2213	message.warning('missing template section: [%s]' % template)
2214
2215
2216	class Section:
2217	"""Static methods and attributes only."""
2218	endtags = [] # Stack of currently open section (level,endtag) tuples.
2219	ids = [] # List of already used ids.
2220	def __init__(self):
2221	raise AssertionError,'no class instances allowed'
2222	@staticmethod
2223	def savetag(level,etag):
2224	"""Save section end."""
2225	Section.endtags.append((level,etag))
2226	@staticmethod
2227	def setlevel(level):
2228	"""Set document level and write open section close tags up to level."""
2229	while Section.endtags and Section.endtags[-1][0] >= level:
2230	writer.write(Section.endtags.pop()[1],trace='section close')
2231	document.level = level
2232	@staticmethod
2233	def gen_id(title):
2234	"""
2235	The normalized value of the id attribute is an NCName according to
2236	the 'Namespaces in XML' Recommendation:
2237	NCName ::= NCNameStartChar NCNameChar*
2238	NCNameChar ::= NameChar - ':'
2239	NCNameStartChar ::= Letter \| '_'
2240	NameChar ::= Letter \| Digit \| '.' \| '-' \| '_' \| ':'
2241	"""
2242	# Replace non-alpha numeric characters in title with underscores and
2243	# convert to lower case.
2244	base_id = re.sub(r'(?u)\W+', '_', char_decode(title)).strip('_').lower()
2245	if 'ascii-ids' in document.attributes:
2246	# Replace non-ASCII characters with ASCII equivalents.
2247	import unicodedata
2248	base_id = unicodedata.normalize('NFKD', base_id).encode('ascii','ignore')
2249	base_id = char_encode(base_id)
2250	# Prefix the ID name with idprefix attribute or underscore if not
2251	# defined. Prefix ensures the ID does not clash with existing IDs.
2252	idprefix = document.attributes.get('idprefix','_')
2253	base_id = idprefix + base_id
2254	i = 1
2255	while True:
2256	if i == 1:
2257	id = base_id
2258	else:
2259	id = '%s_%d' % (base_id, i)
2260	if id not in Section.ids:
2261	Section.ids.append(id)
2262	return id
2263	else:
2264	id = base_id
2265	i += 1
2266	@staticmethod
2267	def set_id():
2268	if not document.attributes.get('sectids') is None \
2269	and 'id' not in AttributeList.attrs:
2270	# Generate ids for sections.
2271	AttributeList.attrs['id'] = Section.gen_id(Title.attributes['title'])
2272	@staticmethod
2273	def translate():
2274	assert Lex.next() is Title
2275	prev_sectname = Title.sectname
2276	Title.translate()
2277	if Title.level == 0 and document.doctype != 'book':
2278	message.error('only book doctypes can contain level 0 sections')
2279	if Title.level > document.level \
2280	and 'basebackend-docbook' in document.attributes \
2281	and prev_sectname in ('colophon','abstract', \
2282	'dedication','glossary','bibliography'):
2283	message.error('%s section cannot contain sub-sections' % prev_sectname)
2284	if Title.level > document.level+1:
2285	# Sub-sections of multi-part book level zero Preface and Appendices
2286	# are meant to be out of sequence.
2287	if document.doctype == 'book' \
2288	and document.level == 0 \
2289	and Title.level == 2 \
2290	and prev_sectname in ('preface','appendix'):
2291	pass
2292	else:
2293	message.warning('section title out of sequence: '
2294	'expected level %d, got level %d'
2295	% (document.level+1, Title.level))
2296	Section.set_id()
2297	Section.setlevel(Title.level)
2298	if 'numbered' in document.attributes:
2299	Title.attributes['sectnum'] = Title.getnumber(document.level)
2300	else:
2301	Title.attributes['sectnum'] = ''
2302	AttributeList.consume(Title.attributes)
2303	stag,etag = config.section2tags(Title.sectname,Title.attributes)
2304	Section.savetag(Title.level,etag)
2305	writer.write(stag,trace='section open: level %d: %s' %
2306	(Title.level, Title.attributes['title']))
2307	Section.translate_body()
2308	@staticmethod
2309	def translate_body(terminator=Title):
2310	isempty = True
2311	next = Lex.next()
2312	while next and next is not terminator:
2313	if isinstance(terminator,DelimitedBlock) and next is Title:
2314	message.error('section title not permitted in delimited block')
2315	next.translate()
2316	next = Lex.next()
2317	isempty = False
2318	# The section is not empty if contains a subsection.
2319	if next and isempty and Title.level > document.level:
2320	isempty = False
2321	# Report empty sections if invalid markup will result.
2322	if isempty:
2323	if document.backend == 'docbook' and Title.sectname != 'index':
2324	message.error('empty section is not valid')
2325
2326	class AbstractBlock:
2327
2328	blocknames = [] # Global stack of names for push_blockname() and pop_blockname().
2329
2330	def __init__(self):
2331	# Configuration parameter names common to all blocks.
2332	self.CONF_ENTRIES = ('delimiter','options','subs','presubs','postsubs',
2333	'posattrs','style','.*-style','template','filter')
2334	self.start = None # File reader cursor at start delimiter.
2335	self.defname=None # Configuration file block definition section name.
2336	# Configuration parameters.
2337	self.delimiter=None # Regular expression matching block delimiter.
2338	self.delimiter_reo=None # Compiled delimiter.
2339	self.template=None # template section entry.
2340	self.options=() # options entry list.
2341	self.presubs=None # presubs/subs entry list.
2342	self.postsubs=() # postsubs entry list.
2343	self.filter=None # filter entry.
2344	self.posattrs=() # posattrs entry list.
2345	self.style=None # Default style.
2346	self.styles=OrderedDict() # Each entry is a styles dictionary.
2347	# Before a block is processed it's attributes (from it's
2348	# attributes list) are merged with the block configuration parameters
2349	# (by self.merge_attributes()) resulting in the template substitution
2350	# dictionary (self.attributes) and the block's processing parameters
2351	# (self.parameters).
2352	self.attributes={}
2353	# The names of block parameters.
2354	self.PARAM_NAMES=('template','options','presubs','postsubs','filter')
2355	self.parameters=None
2356	# Leading delimiter match object.
2357	self.mo=None
2358	def short_name(self):
2359	""" Return the text following the first dash in the section name."""
2360	i = self.defname.find('-')
2361	if i == -1:
2362	return self.defname
2363	else:
2364	return self.defname[i+1:]
2365	def error(self, msg, cursor=None, halt=False):
2366	message.error('[%s] %s' % (self.defname,msg), cursor, halt)
2367	def is_conf_entry(self,param):
2368	"""Return True if param matches an allowed configuration file entry
2369	name."""
2370	for s in self.CONF_ENTRIES:
2371	if re.match('^'+s+'$',param):
2372	return True
2373	return False
2374	def load(self,defname,entries):
2375	"""Update block definition from section 'entries' dictionary."""
2376	self.defname = defname
2377	self.update_parameters(entries, self, all=True)
2378	def update_parameters(self, src, dst=None, all=False):
2379	"""
2380	Parse processing parameters from src dictionary to dst object.
2381	dst defaults to self.parameters.
2382	If all is True then copy src entries that aren't parameter names.
2383	"""
2384	dst = dst or self.parameters
2385	msg = '[%s] malformed entry %%s: %%s' % self.defname
2386	def copy(obj,k,v):
2387	if isinstance(obj,dict):
2388	obj[k] = v
2389	else:
2390	setattr(obj,k,v)
2391	for k,v in src.items():
2392	if not re.match(r'\d+',k) and not is_name(k):
2393	raise EAsciiDoc, msg % (k,v)
2394	if k == 'template':
2395	if not is_name(v):
2396	raise EAsciiDoc, msg % (k,v)
2397	copy(dst,k,v)
2398	elif k == 'filter':
2399	copy(dst,k,v)
2400	elif k == 'options':
2401	if isinstance(v,str):
2402	v = parse_options(v, (), msg % (k,v))
2403	# Merge with existing options.
2404	v = tuple(set(dst.options).union(set(v)))
2405	copy(dst,k,v)
2406	elif k in ('subs','presubs','postsubs'):
2407	# Subs is an alias for presubs.
2408	if k == 'subs': k = 'presubs'
2409	if isinstance(v,str):
2410	v = parse_options(v, SUBS_OPTIONS, msg % (k,v))
2411	copy(dst,k,v)
2412	elif k == 'delimiter':
2413	if v and is_re(v):
2414	copy(dst,k,v)
2415	else:
2416	raise EAsciiDoc, msg % (k,v)
2417	elif k == 'style':
2418	if is_name(v):
2419	copy(dst,k,v)
2420	else:
2421	raise EAsciiDoc, msg % (k,v)
2422	elif k == 'posattrs':
2423	v = parse_options(v, (), msg % (k,v))
2424	copy(dst,k,v)
2425	else:
2426	mo = re.match(r'^(?P<style>.*)-style$',k)
2427	if mo:
2428	if not v:
2429	raise EAsciiDoc, msg % (k,v)
2430	style = mo.group('style')
2431	if not is_name(style):
2432	raise EAsciiDoc, msg % (k,v)
2433	d = {}
2434	if not parse_named_attributes(v,d):
2435	raise EAsciiDoc, msg % (k,v)
2436	if 'subs' in d:
2437	# Subs is an alias for presubs.
2438	d['presubs'] = d['subs']
2439	del d['subs']
2440	self.styles[style] = d
2441	elif all or k in self.PARAM_NAMES:
2442	copy(dst,k,v) # Derived class specific entries.
2443	def get_param(self,name,params=None):
2444	"""
2445	Return named processing parameter from params dictionary.
2446	If the parameter is not in params look in self.parameters.
2447	"""
2448	if params and name in params:
2449	return params[name]
2450	elif name in self.parameters:
2451	return self.parameters[name]
2452	else:
2453	return None
2454	def get_subs(self,params=None):
2455	"""
2456	Return (presubs,postsubs) tuple.
2457	"""
2458	presubs = self.get_param('presubs',params)
2459	postsubs = self.get_param('postsubs',params)
2460	return (presubs,postsubs)
2461	def dump(self):
2462	"""Write block definition to stdout."""
2463	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2464	write('['+self.defname+']')
2465	if self.is_conf_entry('delimiter'):
2466	write('delimiter='+self.delimiter)
2467	if self.template:
2468	write('template='+self.template)
2469	if self.options:
2470	write('options='+','.join(self.options))
2471	if self.presubs:
2472	if self.postsubs:
2473	write('presubs='+','.join(self.presubs))
2474	else:
2475	write('subs='+','.join(self.presubs))
2476	if self.postsubs:
2477	write('postsubs='+','.join(self.postsubs))
2478	if self.filter:
2479	write('filter='+self.filter)
2480	if self.posattrs:
2481	write('posattrs='+','.join(self.posattrs))
2482	if self.style:
2483	write('style='+self.style)
2484	if self.styles:
2485	for style,d in self.styles.items():
2486	s = ''
2487	for k,v in d.items(): s += '%s=%r,' % (k,v)
2488	write('%s-style=%s' % (style,s[:-1]))
2489	def validate(self):
2490	"""Validate block after the complete configuration has been loaded."""
2491	if self.is_conf_entry('delimiter') and not self.delimiter:
2492	raise EAsciiDoc,'[%s] missing delimiter' % self.defname
2493	if self.style:
2494	if not is_name(self.style):
2495	raise EAsciiDoc, 'illegal style name: %s' % self.style
2496	if not self.style in self.styles:
2497	if not isinstance(self,List): # Lists don't have templates.
2498	message.warning('[%s] \'%s\' style not in %s' % (
2499	self.defname,self.style,self.styles.keys()))
2500	# Check all styles for missing templates.
2501	all_styles_have_template = True
2502	for k,v in self.styles.items():
2503	t = v.get('template')
2504	if t and not t in config.sections:
2505	# Defer check if template name contains attributes.
2506	if not re.search(r'{.+}',t):
2507	message.warning('missing template section: [%s]' % t)
2508	if not t:
2509	all_styles_have_template = False
2510	# Check we have a valid template entry or alternatively that all the
2511	# styles have templates.
2512	if self.is_conf_entry('template') and not 'skip' in self.options:
2513	if self.template:
2514	if not self.template in config.sections:
2515	# Defer check if template name contains attributes.
2516	if not re.search(r'{.+}',self.template):
2517	message.warning('missing template section: [%s]'
2518	% self.template)
2519	elif not all_styles_have_template:
2520	if not isinstance(self,List): # Lists don't have templates.
2521	message.warning('missing styles templates: [%s]' % self.defname)
2522	def isnext(self):
2523	"""Check if this block is next in document reader."""
2524	result = False
2525	reader.skip_blank_lines()
2526	if reader.read_next():
2527	if not self.delimiter_reo:
2528	# Cache compiled delimiter optimization.
2529	self.delimiter_reo = re.compile(self.delimiter)
2530	mo = self.delimiter_reo.match(reader.read_next())
2531	if mo:
2532	self.mo = mo
2533	result = True
2534	return result
2535	def translate(self):
2536	"""Translate block from document reader."""
2537	if not self.presubs:
2538	self.presubs = config.subsnormal
2539	if reader.cursor:
2540	self.start = reader.cursor[:]
2541	def push_blockname(self, blockname=None):
2542	'''
2543	On block entry set the 'blockname' attribute.
2544	Only applies to delimited blocks, lists and tables.
2545	'''
2546	if blockname is None:
2547	blockname = self.attributes.get('style', self.short_name()).lower()
2548	trace('push blockname', blockname)
2549	self.blocknames.append(blockname)
2550	document.attributes['blockname'] = blockname
2551	def pop_blockname(self):
2552	'''
2553	On block exits restore previous (parent) 'blockname' attribute or
2554	undefine it if we're no longer inside a block.
2555	'''
2556	assert len(self.blocknames) > 0
2557	blockname = self.blocknames.pop()
2558	trace('pop blockname', blockname)
2559	if len(self.blocknames) == 0:
2560	document.attributes['blockname'] = None
2561	else:
2562	document.attributes['blockname'] = self.blocknames[-1]
2563	def merge_attributes(self,attrs,params=[]):
2564	"""
2565	Use the current block's attribute list (attrs dictionary) to build a
2566	dictionary of block processing parameters (self.parameters) and tag
2567	substitution attributes (self.attributes).
2568
2569	1. Copy the default parameters (self.*) to self.parameters.
2570	self.parameters are used internally to render the current block.
2571	Optional params array of additional parameters.
2572
2573	2. Copy attrs to self.attributes. self.attributes are used for template
2574	and tag substitution in the current block.
2575
2576	3. If a style attribute was specified update self.parameters with the
2577	corresponding style parameters; if there are any style parameters
2578	remaining add them to self.attributes (existing attribute list entries
2579	take precedence).
2580
2581	4. Set named positional attributes in self.attributes if self.posattrs
2582	was specified.
2583
2584	5. Finally self.parameters is updated with any corresponding parameters
2585	specified in attrs.
2586
2587	"""
2588
2589	def check_array_parameter(param):
2590	# Check the parameter is a sequence type.
2591	if not is_array(self.parameters[param]):
2592	message.error('malformed %s parameter: %s' %
2593	(param, self.parameters[param]))
2594	# Revert to default value.
2595	self.parameters[param] = getattr(self,param)
2596
2597	params = list(self.PARAM_NAMES) + params
2598	self.attributes = {}
2599	if self.style:
2600	# If a default style is defined make it available in the template.
2601	self.attributes['style'] = self.style
2602	self.attributes.update(attrs)
2603	# Calculate dynamic block parameters.
2604	# Start with configuration file defaults.
2605	self.parameters = AttrDict()
2606	for name in params:
2607	self.parameters[name] = getattr(self,name)
2608	# Load the selected style attributes.
2609	posattrs = self.posattrs
2610	if posattrs and posattrs[0] == 'style':
2611	# Positional attribute style has highest precedence.
2612	style = self.attributes.get('1')
2613	else:
2614	style = None
2615	if not style:
2616	# Use explicit style attribute, fall back to default style.
2617	style = self.attributes.get('style',self.style)
2618	if style:
2619	if not is_name(style):
2620	message.error('illegal style name: %s' % style)
2621	style = self.style
2622	# Lists have implicit styles and do their own style checks.
2623	elif style not in self.styles and not isinstance(self,List):
2624	message.warning('missing style: [%s]: %s' % (self.defname,style))
2625	style = self.style
2626	if style in self.styles:
2627	self.attributes['style'] = style
2628	for k,v in self.styles[style].items():
2629	if k == 'posattrs':
2630	posattrs = v
2631	elif k in params:
2632	self.parameters[k] = v
2633	elif not k in self.attributes:
2634	# Style attributes don't take precedence over explicit.
2635	self.attributes[k] = v
2636	# Set named positional attributes.
2637	for i,v in enumerate(posattrs):
2638	if str(i+1) in self.attributes:
2639	self.attributes[v] = self.attributes[str(i+1)]
2640	# Override config and style attributes with attribute list attributes.
2641	self.update_parameters(attrs)
2642	check_array_parameter('options')
2643	check_array_parameter('presubs')
2644	check_array_parameter('postsubs')
2645
2646	class AbstractBlocks:
2647	"""List of block definitions."""
2648	PREFIX = '' # Conf file section name prefix set in derived classes.
2649	BLOCK_TYPE = None # Block type set in derived classes.
2650	def __init__(self):
2651	self.current=None
2652	self.blocks = [] # List of Block objects.
2653	self.default = None # Default Block.
2654	self.delimiters = None # Combined delimiters regular expression.
2655	def load(self,sections):
2656	"""Load block definition from 'sections' dictionary."""
2657	for k in sections.keys():
2658	if re.match(r'^'+ self.PREFIX + r'.+$',k):
2659	d = {}
2660	parse_entries(sections.get(k,()),d)
2661	for b in self.blocks:
2662	if b.defname == k:
2663	break
2664	else:
2665	b = self.BLOCK_TYPE()
2666	self.blocks.append(b)
2667	try:
2668	b.load(k,d)
2669	except EAsciiDoc,e:
2670	raise EAsciiDoc,'[%s] %s' % (k,str(e))
2671	def dump(self):
2672	for b in self.blocks:
2673	b.dump()
2674	def isnext(self):
2675	for b in self.blocks:
2676	if b.isnext():
2677	self.current = b
2678	return True;
2679	return False
2680	def validate(self):
2681	"""Validate the block definitions."""
2682	# Validate delimiters and build combined lists delimiter pattern.
2683	delimiters = []
2684	for b in self.blocks:
2685	assert b.__class__ is self.BLOCK_TYPE
2686	b.validate()
2687	if b.delimiter:
2688	delimiters.append(b.delimiter)
2689	self.delimiters = re_join(delimiters)
2690
2691	class Paragraph(AbstractBlock):
2692	def __init__(self):
2693	AbstractBlock.__init__(self)
2694	self.text=None # Text in first line of paragraph.
2695	def load(self,name,entries):
2696	AbstractBlock.load(self,name,entries)
2697	def dump(self):
2698	AbstractBlock.dump(self)
2699	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2700	write('')
2701	def isnext(self):
2702	result = AbstractBlock.isnext(self)
2703	if result:
2704	self.text = self.mo.groupdict().get('text')
2705	return result
2706	def translate(self):
2707	AbstractBlock.translate(self)
2708	attrs = self.mo.groupdict().copy()
2709	if 'text' in attrs: del attrs['text']
2710	BlockTitle.consume(attrs)
2711	AttributeList.consume(attrs)
2712	self.merge_attributes(attrs)
2713	reader.read() # Discard (already parsed item first line).
2714	body = reader.read_until(paragraphs.terminators)
2715	if 'skip' in self.parameters.options:
2716	return
2717	body = [self.text] + list(body)
2718	presubs = self.parameters.presubs
2719	postsubs = self.parameters.postsubs
2720	if document.attributes.get('plaintext') is None:
2721	body = Lex.set_margin(body) # Move body to left margin.
2722	body = Lex.subs(body,presubs)
2723	template = self.parameters.template
2724	template = subs_attrs(template,attrs)
2725	stag = config.section2tags(template, self.attributes,skipend=True)[0]
2726	if self.parameters.filter:
2727	body = filter_lines(self.parameters.filter,body,self.attributes)
2728	body = Lex.subs(body,postsubs)
2729	etag = config.section2tags(template, self.attributes,skipstart=True)[1]
2730	# Write start tag, content, end tag.
2731	writer.write(dovetail_tags(stag,body,etag),trace='paragraph')
2732
2733	class Paragraphs(AbstractBlocks):
2734	"""List of paragraph definitions."""
2735	BLOCK_TYPE = Paragraph
2736	PREFIX = 'paradef-'
2737	def __init__(self):
2738	AbstractBlocks.__init__(self)
2739	self.terminators=None # List of compiled re's.
2740	def initialize(self):
2741	self.terminators = [
2742	re.compile(r'^\+$\|^$'),
2743	re.compile(AttributeList.pattern),
2744	re.compile(blocks.delimiters),
2745	re.compile(tables.delimiters),
2746	re.compile(tables_OLD.delimiters),
2747	]
2748	def load(self,sections):
2749	AbstractBlocks.load(self,sections)
2750	def validate(self):
2751	AbstractBlocks.validate(self)
2752	# Check we have a default paragraph definition, put it last in list.
2753	for b in self.blocks:
2754	if b.defname == 'paradef-default':
2755	self.blocks.append(b)
2756	self.default = b
2757	self.blocks.remove(b)
2758	break
2759	else:
2760	raise EAsciiDoc,'missing section: [paradef-default]'
2761
2762	class List(AbstractBlock):
2763	NUMBER_STYLES= ('arabic','loweralpha','upperalpha','lowerroman',
2764	'upperroman')
2765	def __init__(self):
2766	AbstractBlock.__init__(self)
2767	self.CONF_ENTRIES += ('type','tags')
2768	self.PARAM_NAMES += ('tags',)
2769	# listdef conf file parameters.
2770	self.type=None
2771	self.tags=None # Name of listtags-<tags> conf section.
2772	# Calculated parameters.
2773	self.tag=None # Current tags AttrDict.
2774	self.label=None # List item label (labeled lists).
2775	self.text=None # Text in first line of list item.
2776	self.index=None # Matched delimiter 'index' group (numbered lists).
2777	self.type=None # List type ('numbered','bulleted','labeled').
2778	self.ordinal=None # Current list item ordinal number (1..)
2779	self.number_style=None # Current numbered list style ('arabic'..)
2780	def load(self,name,entries):
2781	AbstractBlock.load(self,name,entries)
2782	def dump(self):
2783	AbstractBlock.dump(self)
2784	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
2785	write('type='+self.type)
2786	write('tags='+self.tags)
2787	write('')
2788	def validate(self):
2789	AbstractBlock.validate(self)
2790	tags = [self.tags]
2791	tags += [s['tags'] for s in self.styles.values() if 'tags' in s]
2792	for t in tags:
2793	if t not in lists.tags:
2794	self.error('missing section: [listtags-%s]' % t,halt=True)
2795	def isnext(self):
2796	result = AbstractBlock.isnext(self)
2797	if result:
2798	self.label = self.mo.groupdict().get('label')
2799	self.text = self.mo.groupdict().get('text')
2800	self.index = self.mo.groupdict().get('index')
2801	return result
2802	def translate_entry(self):
2803	assert self.type == 'labeled'
2804	entrytag = subs_tag(self.tag.entry, self.attributes)
2805	labeltag = subs_tag(self.tag.label, self.attributes)
2806	writer.write(entrytag[0],trace='list entry open')
2807	writer.write(labeltag[0],trace='list label open')
2808	# Write labels.
2809	while Lex.next() is self:
2810	reader.read() # Discard (already parsed item first line).
2811	writer.write_tag(self.tag.term, [self.label],
2812	self.presubs, self.attributes,trace='list term')
2813	if self.text: break
2814	writer.write(labeltag[1],trace='list label close')
2815	# Write item text.
2816	self.translate_item()
2817	writer.write(entrytag[1],trace='list entry close')
2818	def translate_item(self):
2819	if self.type == 'callout':
2820	self.attributes['coids'] = calloutmap.calloutids(self.ordinal)
2821	itemtag = subs_tag(self.tag.item, self.attributes)
2822	writer.write(itemtag[0],trace='list item open')
2823	# Write ItemText.
2824	text = reader.read_until(lists.terminators)
2825	if self.text:
2826	text = [self.text] + list(text)
2827	if text:
2828	writer.write_tag(self.tag.text, text, self.presubs, self.attributes,trace='list text')
2829	# Process explicit and implicit list item continuations.
2830	while True:
2831	continuation = reader.read_next() == '+'
2832	if continuation: reader.read() # Discard continuation line.
2833	while Lex.next() in (BlockTitle,AttributeList):
2834	# Consume continued element title and attributes.
2835	Lex.next().translate()
2836	if not continuation and BlockTitle.title:
2837	# Titled elements terminate the list.
2838	break
2839	next = Lex.next()
2840	if next in lists.open:
2841	break
2842	elif isinstance(next,List):
2843	next.translate()
2844	elif isinstance(next,Paragraph) and 'listelement' in next.options:
2845	next.translate()
2846	elif continuation:
2847	# This is where continued elements are processed.
2848	if next is Title:
2849	message.error('section title not allowed in list item',halt=True)
2850	next.translate()
2851	else:
2852	break
2853	writer.write(itemtag[1],trace='list item close')
2854
2855	@staticmethod
2856	def calc_style(index):
2857	"""Return the numbered list style ('arabic'...) of the list item index.
2858	Return None if unrecognized style."""
2859	if re.match(r'^\d+[\.>]$', index):
2860	style = 'arabic'
2861	elif re.match(r'^[ivx]+\)$', index):
2862	style = 'lowerroman'
2863	elif re.match(r'^[IVX]+\)$', index):
2864	style = 'upperroman'
2865	elif re.match(r'^[a-z]\.$', index):
2866	style = 'loweralpha'
2867	elif re.match(r'^[A-Z]\.$', index):
2868	style = 'upperalpha'
2869	else:
2870	assert False
2871	return style
2872
2873	@staticmethod
2874	def calc_index(index,style):
2875	"""Return the ordinal number of (1...) of the list item index
2876	for the given list style."""
2877	def roman_to_int(roman):
2878	roman = roman.lower()
2879	digits = {'i':1,'v':5,'x':10}
2880	result = 0
2881	for i in range(len(roman)):
2882	digit = digits[roman[i]]
2883	# If next digit is larger this digit is negative.
2884	if i+1 < len(roman) and digits[roman[i+1]] > digit:
2885	result -= digit
2886	else:
2887	result += digit
2888	return result
2889	index = index[:-1]
2890	if style == 'arabic':
2891	ordinal = int(index)
2892	elif style == 'lowerroman':
2893	ordinal = roman_to_int(index)
2894	elif style == 'upperroman':
2895	ordinal = roman_to_int(index)
2896	elif style == 'loweralpha':
2897	ordinal = ord(index) - ord('a') + 1
2898	elif style == 'upperalpha':
2899	ordinal = ord(index) - ord('A') + 1
2900	else:
2901	assert False
2902	return ordinal
2903
2904	def check_index(self):
2905	"""Check calculated self.ordinal (1,2,...) against the item number
2906	in the document (self.index) and check the number style is the same as
2907	the first item (self.number_style)."""
2908	assert self.type in ('numbered','callout')
2909	if self.index:
2910	style = self.calc_style(self.index)
2911	if style != self.number_style:
2912	message.warning('list item style: expected %s got %s' %
2913	(self.number_style,style), offset=1)
2914	ordinal = self.calc_index(self.index,style)
2915	if ordinal != self.ordinal:
2916	message.warning('list item index: expected %s got %s' %
2917	(self.ordinal,ordinal), offset=1)
2918
2919	def check_tags(self):
2920	""" Check that all necessary tags are present. """
2921	tags = set(Lists.TAGS)
2922	if self.type != 'labeled':
2923	tags = tags.difference(['entry','label','term'])
2924	missing = tags.difference(self.tag.keys())
2925	if missing:
2926	self.error('missing tag(s): %s' % ','.join(missing), halt=True)
2927	def translate(self):
2928	AbstractBlock.translate(self)
2929	if self.short_name() in ('bibliography','glossary','qanda'):
2930	message.deprecated('old %s list syntax' % self.short_name())
2931	lists.open.append(self)
2932	attrs = self.mo.groupdict().copy()
2933	for k in ('label','text','index'):
2934	if k in attrs: del attrs[k]
2935	if self.index:
2936	# Set the numbering style from first list item.
2937	attrs['style'] = self.calc_style(self.index)
2938	BlockTitle.consume(attrs)
2939	AttributeList.consume(attrs)
2940	self.merge_attributes(attrs,['tags'])
2941	self.push_blockname()
2942	if self.type in ('numbered','callout'):
2943	self.number_style = self.attributes.get('style')
2944	if self.number_style not in self.NUMBER_STYLES:
2945	message.error('illegal numbered list style: %s' % self.number_style)
2946	# Fall back to default style.
2947	self.attributes['style'] = self.number_style = self.style
2948	self.tag = lists.tags[self.parameters.tags]
2949	self.check_tags()
2950	if 'width' in self.attributes:
2951	# Set horizontal list 'labelwidth' and 'itemwidth' attributes.
2952	v = str(self.attributes['width'])
2953	mo = re.match(r'^(\d{1,2})%?$',v)
2954	if mo:
2955	labelwidth = int(mo.group(1))
2956	self.attributes['labelwidth'] = str(labelwidth)
2957	self.attributes['itemwidth'] = str(100-labelwidth)
2958	else:
2959	self.error('illegal attribute value: width="%s"' % v)
2960	stag,etag = subs_tag(self.tag.list, self.attributes)
2961	if stag:
2962	writer.write(stag,trace='list open')
2963	self.ordinal = 0
2964	# Process list till list syntax changes or there is a new title.
2965	while Lex.next() is self and not BlockTitle.title:
2966	self.ordinal += 1
2967	document.attributes['listindex'] = str(self.ordinal)
2968	if self.type in ('numbered','callout'):
2969	self.check_index()
2970	if self.type in ('bulleted','numbered','callout'):
2971	reader.read() # Discard (already parsed item first line).
2972	self.translate_item()
2973	elif self.type == 'labeled':
2974	self.translate_entry()
2975	else:
2976	raise AssertionError,'illegal [%s] list type' % self.defname
2977	if etag:
2978	writer.write(etag,trace='list close')
2979	if self.type == 'callout':
2980	calloutmap.validate(self.ordinal)
2981	calloutmap.listclose()
2982	lists.open.pop()
2983	if len(lists.open):
2984	document.attributes['listindex'] = str(lists.open[-1].ordinal)
2985	self.pop_blockname()
2986
2987	class Lists(AbstractBlocks):
2988	"""List of List objects."""
2989	BLOCK_TYPE = List
2990	PREFIX = 'listdef-'
2991	TYPES = ('bulleted','numbered','labeled','callout')
2992	TAGS = ('list', 'entry','item','text', 'label','term')
2993	def __init__(self):
2994	AbstractBlocks.__init__(self)
2995	self.open = [] # A stack of the current and parent lists.
2996	self.tags={} # List tags dictionary. Each entry is a tags AttrDict.
2997	self.terminators=None # List of compiled re's.
2998	def initialize(self):
2999	self.terminators = [
3000	re.compile(r'^\+$\|^$'),
3001	re.compile(AttributeList.pattern),
3002	re.compile(lists.delimiters),
3003	re.compile(blocks.delimiters),
3004	re.compile(tables.delimiters),
3005	re.compile(tables_OLD.delimiters),
3006	]
3007	def load(self,sections):
3008	AbstractBlocks.load(self,sections)
3009	self.load_tags(sections)
3010	def load_tags(self,sections):
3011	"""
3012	Load listtags-* conf file sections to self.tags.
3013	"""
3014	for section in sections.keys():
3015	mo = re.match(r'^listtags-(?P<name>\w+)$',section)
3016	if mo:
3017	name = mo.group('name')
3018	if name in self.tags:
3019	d = self.tags[name]
3020	else:
3021	d = AttrDict()
3022	parse_entries(sections.get(section,()),d)
3023	for k in d.keys():
3024	if k not in self.TAGS:
3025	message.warning('[%s] contains illegal list tag: %s' %
3026	(section,k))
3027	self.tags[name] = d
3028	def validate(self):
3029	AbstractBlocks.validate(self)
3030	for b in self.blocks:
3031	# Check list has valid type.
3032	if not b.type in Lists.TYPES:
3033	raise EAsciiDoc,'[%s] illegal type' % b.defname
3034	b.validate()
3035	def dump(self):
3036	AbstractBlocks.dump(self)
3037	for k,v in self.tags.items():
3038	dump_section('listtags-'+k, v)
3039
3040
3041	class DelimitedBlock(AbstractBlock):
3042	def __init__(self):
3043	AbstractBlock.__init__(self)
3044	def load(self,name,entries):
3045	AbstractBlock.load(self,name,entries)
3046	def dump(self):
3047	AbstractBlock.dump(self)
3048	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
3049	write('')
3050	def isnext(self):
3051	return AbstractBlock.isnext(self)
3052	def translate(self):
3053	AbstractBlock.translate(self)
3054	reader.read() # Discard delimiter.
3055	self.merge_attributes(AttributeList.attrs)
3056	if not 'skip' in self.parameters.options:
3057	BlockTitle.consume(self.attributes)
3058	AttributeList.consume()
3059	self.push_blockname()
3060	options = self.parameters.options
3061	if 'skip' in options:
3062	reader.read_until(self.delimiter,same_file=True)
3063	elif safe() and self.defname == 'blockdef-backend':
3064	message.unsafe('Backend Block')
3065	reader.read_until(self.delimiter,same_file=True)
3066	else:
3067	template = self.parameters.template
3068	template = subs_attrs(template,self.attributes)
3069	name = self.short_name()+' block'
3070	if 'sectionbody' in options:
3071	# The body is treated like a section body.
3072	stag,etag = config.section2tags(template,self.attributes)
3073	writer.write(stag,trace=name+' open')
3074	Section.translate_body(self)
3075	writer.write(etag,trace=name+' close')
3076	else:
3077	stag = config.section2tags(template,self.attributes,skipend=True)[0]
3078	body = reader.read_until(self.delimiter,same_file=True)
3079	presubs = self.parameters.presubs
3080	postsubs = self.parameters.postsubs
3081	body = Lex.subs(body,presubs)
3082	if self.parameters.filter:
3083	body = filter_lines(self.parameters.filter,body,self.attributes)
3084	body = Lex.subs(body,postsubs)
3085	# Write start tag, content, end tag.
3086	etag = config.section2tags(template,self.attributes,skipstart=True)[1]
3087	writer.write(dovetail_tags(stag,body,etag),trace=name)
3088	trace(self.short_name()+' block close',etag)
3089	if reader.eof():
3090	self.error('missing closing delimiter',self.start)
3091	else:
3092	delimiter = reader.read() # Discard delimiter line.
3093	assert re.match(self.delimiter,delimiter)
3094	self.pop_blockname()
3095
3096	class DelimitedBlocks(AbstractBlocks):
3097	"""List of delimited blocks."""
3098	BLOCK_TYPE = DelimitedBlock
3099	PREFIX = 'blockdef-'
3100	def __init__(self):
3101	AbstractBlocks.__init__(self)
3102	def load(self,sections):
3103	"""Update blocks defined in 'sections' dictionary."""
3104	AbstractBlocks.load(self,sections)
3105	def validate(self):
3106	AbstractBlocks.validate(self)
3107
3108	class Column:
3109	"""Table column."""
3110	def __init__(self, width=None, align_spec=None, style=None):
3111	self.width = width or '1'
3112	self.halign, self.valign = Table.parse_align_spec(align_spec)
3113	self.style = style # Style name or None.
3114	# Calculated attribute values.
3115	self.abswidth = None # 1.. (page units).
3116	self.pcwidth = None # 1..99 (percentage).
3117
3118	class Cell:
3119	def __init__(self, data, span_spec=None, align_spec=None, style=None):
3120	self.data = data
3121	self.span, self.vspan = Table.parse_span_spec(span_spec)
3122	self.halign, self.valign = Table.parse_align_spec(align_spec)
3123	self.style = style
3124	self.reserved = False
3125	def __repr__(self):
3126	return '<Cell: %d.%d %s.%s %s "%s">' % (
3127	self.span, self.vspan,
3128	self.halign, self.valign,
3129	self.style or '',
3130	self.data)
3131	def clone_reserve(self):
3132	"""Return a clone of self to reserve vertically spanned cell."""
3133	result = copy.copy(self)
3134	result.vspan = 1
3135	result.reserved = True
3136	return result
3137
3138	class Table(AbstractBlock):
3139	ALIGN = {'<':'left', '>':'right', '^':'center'}
3140	VALIGN = {'<':'top', '>':'bottom', '^':'middle'}
3141	FORMATS = ('psv','csv','dsv')
3142	SEPARATORS = dict(
3143	csv=',',
3144	dsv=r':\|\n',
3145	# The count and align group matches are not exact.
3146	psv=r'((?<!\S)((?P<span>[\d.]+)(?P<op>[*+]))?(?P<align>[<\^>.]{,3})?(?P<style>[a-z])?)?\\|'
3147	)
3148	def __init__(self):
3149	AbstractBlock.__init__(self)
3150	self.CONF_ENTRIES += ('format','tags','separator')
3151	# tabledef conf file parameters.
3152	self.format='psv'
3153	self.separator=None
3154	self.tags=None # Name of tabletags-<tags> conf section.
3155	# Calculated parameters.
3156	self.abswidth=None # 1.. (page units).
3157	self.pcwidth = None # 1..99 (percentage).
3158	self.rows=[] # Parsed rows, each row is a list of Cells.
3159	self.columns=[] # List of Columns.
3160	@staticmethod
3161	def parse_align_spec(align_spec):
3162	"""
3163	Parse AsciiDoc cell alignment specifier and return 2-tuple with
3164	horizonatal and vertical alignment names. Unspecified alignments
3165	set to None.
3166	"""
3167	result = (None, None)
3168	if align_spec:
3169	mo = re.match(r'^([<\^>])?(\.([<\^>]))?$', align_spec)
3170	if mo:
3171	result = (Table.ALIGN.get(mo.group(1)),
3172	Table.VALIGN.get(mo.group(3)))
3173	return result
3174	@staticmethod
3175	def parse_span_spec(span_spec):
3176	"""
3177	Parse AsciiDoc cell span specifier and return 2-tuple with horizonatal
3178	and vertical span counts. Set default values (1,1) if not
3179	specified.
3180	"""
3181	result = (None, None)
3182	if span_spec:
3183	mo = re.match(r'^(\d+)?(\.(\d+))?$', span_spec)
3184	if mo:
3185	result = (mo.group(1) and int(mo.group(1)),
3186	mo.group(3) and int(mo.group(3)))
3187	return (result[0] or 1, result[1] or 1)
3188	def load(self,name,entries):
3189	AbstractBlock.load(self,name,entries)
3190	def dump(self):
3191	AbstractBlock.dump(self)
3192	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
3193	write('format='+self.format)
3194	write('')
3195	def validate(self):
3196	AbstractBlock.validate(self)
3197	if self.format not in Table.FORMATS:
3198	self.error('illegal format=%s' % self.format,halt=True)
3199	self.tags = self.tags or 'default'
3200	tags = [self.tags]
3201	tags += [s['tags'] for s in self.styles.values() if 'tags' in s]
3202	for t in tags:
3203	if t not in tables.tags:
3204	self.error('missing section: [tabletags-%s]' % t,halt=True)
3205	if self.separator:
3206	# Evaluate escape characters.
3207	self.separator = literal_eval('"'+self.separator+'"')
3208	#TODO: Move to class Tables
3209	# Check global table parameters.
3210	elif config.pagewidth is None:
3211	self.error('missing [miscellaneous] entry: pagewidth')
3212	elif config.pageunits is None:
3213	self.error('missing [miscellaneous] entry: pageunits')
3214	def validate_attributes(self):
3215	"""Validate and parse table attributes."""
3216	# Set defaults.
3217	format = self.format
3218	tags = self.tags
3219	separator = self.separator
3220	abswidth = float(config.pagewidth)
3221	pcwidth = 100.0
3222	for k,v in self.attributes.items():
3223	if k == 'format':
3224	if v not in self.FORMATS:
3225	self.error('illegal %s=%s' % (k,v))
3226	else:
3227	format = v
3228	elif k == 'tags':
3229	if v not in tables.tags:
3230	self.error('illegal %s=%s' % (k,v))
3231	else:
3232	tags = v
3233	elif k == 'separator':
3234	separator = v
3235	elif k == 'width':
3236	if not re.match(r'^\d{1,3}%$',v) or int(v[:-1]) > 100:
3237	self.error('illegal %s=%s' % (k,v))
3238	else:
3239	abswidth = float(v[:-1])/100 * config.pagewidth
3240	pcwidth = float(v[:-1])
3241	# Calculate separator if it has not been specified.
3242	if not separator:
3243	separator = Table.SEPARATORS[format]
3244	if format == 'csv':
3245	if len(separator) > 1:
3246	self.error('illegal csv separator=%s' % separator)
3247	separator = ','
3248	else:
3249	if not is_re(separator):
3250	self.error('illegal regular expression: separator=%s' %
3251	separator)
3252	self.parameters.format = format
3253	self.parameters.tags = tags
3254	self.parameters.separator = separator
3255	self.abswidth = abswidth
3256	self.pcwidth = pcwidth
3257	def get_tags(self,params):
3258	tags = self.get_param('tags',params)
3259	assert(tags and tags in tables.tags)
3260	return tables.tags[tags]
3261	def get_style(self,prefix):
3262	"""
3263	Return the style dictionary whose name starts with 'prefix'.
3264	"""
3265	if prefix is None:
3266	return None
3267	names = self.styles.keys()
3268	names.sort()
3269	for name in names:
3270	if name.startswith(prefix):
3271	return self.styles[name]
3272	else:
3273	self.error('missing style: %s*' % prefix)
3274	return None
3275	def parse_cols(self, cols, halign, valign):
3276	"""
3277	Build list of column objects from table 'cols', 'halign' and 'valign'
3278	attributes.
3279	"""
3280	# [<multiplier>*][<align>][<width>][<style>]
3281	COLS_RE1 = r'^((?P<count>\d+)\)?(?P<align>[<\^>.]{,3})?(?P<width>\d+%?)?(?P<style>[a-z]\w)?$'
3282	# [<multiplier>*][<width>][<align>][<style>]
3283	COLS_RE2 = r'^((?P<count>\d+)\)?(?P<width>\d+%?)?(?P<align>[<\^>.]{,3})?(?P<style>[a-z]\w)?$'
3284	reo1 = re.compile(COLS_RE1)
3285	reo2 = re.compile(COLS_RE2)
3286	cols = str(cols)
3287	if re.match(r'^\d+$',cols):
3288	for i in range(int(cols)):
3289	self.columns.append(Column())
3290	else:
3291	for col in re.split(r'\s,\s',cols):
3292	mo = reo1.match(col)
3293	if not mo:
3294	mo = reo2.match(col)
3295	if mo:
3296	count = int(mo.groupdict().get('count') or 1)
3297	for i in range(count):
3298	self.columns.append(
3299	Column(mo.group('width'), mo.group('align'),
3300	self.get_style(mo.group('style')))
3301	)
3302	else:
3303	self.error('illegal column spec: %s' % col,self.start)
3304	# Set column (and indirectly cell) default alignments.
3305	for col in self.columns:
3306	col.halign = col.halign or halign or document.attributes.get('halign') or 'left'
3307	col.valign = col.valign or valign or document.attributes.get('valign') or 'top'
3308	# Validate widths and calculate missing widths.
3309	n = 0; percents = 0; props = 0
3310	for col in self.columns:
3311	if col.width:
3312	if col.width[-1] == '%': percents += int(col.width[:-1])
3313	else: props += int(col.width)
3314	n += 1
3315	if percents > 0 and props > 0:
3316	self.error('mixed percent and proportional widths: %s'
3317	% cols,self.start)
3318	pcunits = percents > 0
3319	# Fill in missing widths.
3320	if n < len(self.columns) and percents < 100:
3321	if pcunits:
3322	width = float(100 - percents)/float(len(self.columns) - n)
3323	else:
3324	width = 1
3325	for col in self.columns:
3326	if not col.width:
3327	if pcunits:
3328	col.width = str(int(width))+'%'
3329	percents += width
3330	else:
3331	col.width = str(width)
3332	props += width
3333	# Calculate column alignment and absolute and percent width values.
3334	percents = 0
3335	for col in self.columns:
3336	if pcunits:
3337	col.pcwidth = float(col.width[:-1])
3338	else:
3339	col.pcwidth = (float(col.width)/props)*100
3340	col.abswidth = self.abswidth * (col.pcwidth/100)
3341	if config.pageunits in ('cm','mm','in','em'):
3342	col.abswidth = '%.2f' % round(col.abswidth,2)
3343	else:
3344	col.abswidth = '%d' % round(col.abswidth)
3345	percents += col.pcwidth
3346	col.pcwidth = int(col.pcwidth)
3347	if round(percents) > 100:
3348	self.error('total width exceeds 100%%: %s' % cols,self.start)
3349	elif round(percents) < 100:
3350	self.error('total width less than 100%%: %s' % cols,self.start)
3351	def build_colspecs(self):
3352	"""
3353	Generate column related substitution attributes.
3354	"""
3355	cols = []
3356	i = 1
3357	for col in self.columns:
3358	colspec = self.get_tags(col.style).colspec
3359	if colspec:
3360	self.attributes['halign'] = col.halign
3361	self.attributes['valign'] = col.valign
3362	self.attributes['colabswidth'] = col.abswidth
3363	self.attributes['colpcwidth'] = col.pcwidth
3364	self.attributes['colnumber'] = str(i)
3365	s = subs_attrs(colspec, self.attributes)
3366	if not s:
3367	message.warning('colspec dropped: contains undefined attribute')
3368	else:
3369	cols.append(s)
3370	i += 1
3371	if cols:
3372	self.attributes['colspecs'] = writer.newline.join(cols)
3373	def parse_rows(self, text):
3374	"""
3375	Parse the table source text into self.rows (a list of rows, each row
3376	is a list of Cells.
3377	"""
3378	reserved = {} # Reserved cells generated by rowspans.
3379	if self.parameters.format in ('psv','dsv'):
3380	colcount = len(self.columns)
3381	parsed_cells = self.parse_psv_dsv(text)
3382	ri = 0 # Current row index 0..
3383	ci = 0 # Column counter 0..colcount
3384	row = []
3385	i = 0
3386	while True:
3387	resv = reserved.get(ri) and reserved[ri].get(ci)
3388	if resv:
3389	# We have a cell generated by a previous row span so
3390	# process it before continuing with the current parsed
3391	# cell.
3392	cell = resv
3393	else:
3394	if i >= len(parsed_cells):
3395	break # No more parsed or reserved cells.
3396	cell = parsed_cells[i]
3397	i += 1
3398	if cell.vspan > 1:
3399	# Generate ensuing reserved cells spanned vertically by
3400	# the current cell.
3401	for j in range(1, cell.vspan):
3402	if not ri+j in reserved:
3403	reserved[ri+j] = {}
3404	reserved[ri+j][ci] = cell.clone_reserve()
3405	ci += cell.span
3406	if ci <= colcount:
3407	row.append(cell)
3408	if ci >= colcount:
3409	self.rows.append(row)
3410	ri += 1
3411	row = []
3412	ci = 0
3413	elif self.parameters.format == 'csv':
3414	self.rows = self.parse_csv(text)
3415	else:
3416	assert True,'illegal table format'
3417	# Check for empty rows containing only reserved (spanned) cells.
3418	for ri,row in enumerate(self.rows):
3419	empty = True
3420	for cell in row:
3421	if not cell.reserved:
3422	empty = False
3423	break
3424	if empty:
3425	message.warning('table row %d: empty spanned row' % (ri+1))
3426	# Check that all row spans match.
3427	for ri,row in enumerate(self.rows):
3428	row_span = 0
3429	for cell in row:
3430	row_span += cell.span
3431	if ri == 0:
3432	header_span = row_span
3433	if row_span < header_span:
3434	message.warning('table row %d: does not span all columns' % (ri+1))
3435	if row_span > header_span:
3436	message.warning('table row %d: exceeds columns span' % (ri+1))
3437	def subs_rows(self, rows, rowtype='body'):
3438	"""
3439	Return a string of output markup from a list of rows, each row
3440	is a list of raw data text.
3441	"""
3442	tags = tables.tags[self.parameters.tags]
3443	if rowtype == 'header':
3444	rtag = tags.headrow
3445	elif rowtype == 'footer':
3446	rtag = tags.footrow
3447	else:
3448	rtag = tags.bodyrow
3449	result = []
3450	stag,etag = subs_tag(rtag,self.attributes)
3451	for row in rows:
3452	result.append(stag)
3453	result += self.subs_row(row,rowtype)
3454	result.append(etag)
3455	return writer.newline.join(result)
3456	def subs_row(self, row, rowtype):
3457	"""
3458	Substitute the list of Cells using the data tag.
3459	Returns a list of marked up table cell elements.
3460	"""
3461	result = []
3462	i = 0
3463	for cell in row:
3464	if cell.reserved:
3465	# Skip vertically spanned placeholders.
3466	i += cell.span
3467	continue
3468	if i >= len(self.columns):
3469	break # Skip cells outside the header width.
3470	col = self.columns[i]
3471	self.attributes['halign'] = cell.halign or col.halign
3472	self.attributes['valign'] = cell.valign or col.valign
3473	self.attributes['colabswidth'] = col.abswidth
3474	self.attributes['colpcwidth'] = col.pcwidth
3475	self.attributes['colnumber'] = str(i+1)
3476	self.attributes['colspan'] = str(cell.span)
3477	self.attributes['colstart'] = self.attributes['colnumber']
3478	self.attributes['colend'] = str(i+cell.span)
3479	self.attributes['rowspan'] = str(cell.vspan)
3480	self.attributes['morerows'] = str(cell.vspan-1)
3481	# Fill missing column data with blanks.
3482	if i > len(self.columns) - 1:
3483	data = ''
3484	else:
3485	data = cell.data
3486	if rowtype == 'header':
3487	# Use table style unless overriden by cell style.
3488	colstyle = cell.style
3489	else:
3490	# If the cell style is not defined use the column style.
3491	colstyle = cell.style or col.style
3492	tags = self.get_tags(colstyle)
3493	presubs,postsubs = self.get_subs(colstyle)
3494	data = [data]
3495	data = Lex.subs(data, presubs)
3496	data = filter_lines(self.get_param('filter',colstyle),
3497	data, self.attributes)
3498	data = Lex.subs(data, postsubs)
3499	if rowtype != 'header':
3500	ptag = tags.paragraph
3501	if ptag:
3502	stag,etag = subs_tag(ptag,self.attributes)
3503	text = '\n'.join(data).strip()
3504	data = []
3505	for para in re.split(r'\n{2,}',text):
3506	data += dovetail_tags([stag],para.split('\n'),[etag])
3507	if rowtype == 'header':
3508	dtag = tags.headdata
3509	elif rowtype == 'footer':
3510	dtag = tags.footdata
3511	else:
3512	dtag = tags.bodydata
3513	stag,etag = subs_tag(dtag,self.attributes)
3514	result = result + dovetail_tags([stag],data,[etag])
3515	i += cell.span
3516	return result
3517	def parse_csv(self,text):
3518	"""
3519	Parse the table source text and return a list of rows, each row
3520	is a list of Cells.
3521	"""
3522	import StringIO
3523	import csv
3524	rows = []
3525	rdr = csv.reader(StringIO.StringIO('\r\n'.join(text)),
3526	delimiter=self.parameters.separator, skipinitialspace=True)
3527	try:
3528	for row in rdr:
3529	rows.append([Cell(data) for data in row])
3530	except Exception:
3531	self.error('csv parse error: %s' % row)
3532	return rows
3533	def parse_psv_dsv(self,text):
3534	"""
3535	Parse list of PSV or DSV table source text lines and return a list of
3536	Cells.
3537	"""
3538	def append_cell(data, span_spec, op, align_spec, style):
3539	op = op or '+'
3540	if op == '*': # Cell multiplier.
3541	span = Table.parse_span_spec(span_spec)[0]
3542	for i in range(span):
3543	cells.append(Cell(data, '1', align_spec, style))
3544	elif op == '+': # Column spanner.
3545	cells.append(Cell(data, span_spec, align_spec, style))
3546	else:
3547	self.error('illegal table cell operator')
3548	text = '\n'.join(text)
3549	separator = '(?msu)'+self.parameters.separator
3550	format = self.parameters.format
3551	start = 0
3552	span = None
3553	op = None
3554	align = None
3555	style = None
3556	cells = []
3557	data = ''
3558	for mo in re.finditer(separator,text):
3559	data += text[start:mo.start()]
3560	if data.endswith('\\'):
3561	data = data[:-1]+mo.group() # Reinstate escaped separators.
3562	else:
3563	append_cell(data, span, op, align, style)
3564	span = mo.groupdict().get('span')
3565	op = mo.groupdict().get('op')
3566	align = mo.groupdict().get('align')
3567	style = mo.groupdict().get('style')
3568	if style:
3569	style = self.get_style(style)
3570	data = ''
3571	start = mo.end()
3572	# Last cell follows final separator.
3573	data += text[start:]
3574	append_cell(data, span, op, align, style)
3575	# We expect a dummy blank item preceeding first PSV cell.
3576	if format == 'psv':
3577	if cells[0].data.strip() != '':
3578	self.error('missing leading separator: %s' % separator,
3579	self.start)
3580	else:
3581	cells.pop(0)
3582	return cells
3583	def translate(self):
3584	AbstractBlock.translate(self)
3585	reader.read() # Discard delimiter.
3586	# Reset instance specific properties.
3587	self.columns = []
3588	self.rows = []
3589	attrs = {}
3590	BlockTitle.consume(attrs)
3591	# Mix in document attribute list.
3592	AttributeList.consume(attrs)
3593	self.merge_attributes(attrs)
3594	self.validate_attributes()
3595	# Add global and calculated configuration parameters.
3596	self.attributes['pagewidth'] = config.pagewidth
3597	self.attributes['pageunits'] = config.pageunits
3598	self.attributes['tableabswidth'] = int(self.abswidth)
3599	self.attributes['tablepcwidth'] = int(self.pcwidth)
3600	# Read the entire table.
3601	text = reader.read_until(self.delimiter)
3602	if reader.eof():
3603	self.error('missing closing delimiter',self.start)
3604	else:
3605	delimiter = reader.read() # Discard closing delimiter.
3606	assert re.match(self.delimiter,delimiter)
3607	if len(text) == 0:
3608	message.warning('[%s] table is empty' % self.defname)
3609	return
3610	self.push_blockname('table')
3611	cols = attrs.get('cols')
3612	if not cols:
3613	# Calculate column count from number of items in first line.
3614	if self.parameters.format == 'csv':
3615	cols = text[0].count(self.parameters.separator) + 1
3616	else:
3617	cols = 0
3618	for cell in self.parse_psv_dsv(text[:1]):
3619	cols += cell.span
3620	self.parse_cols(cols, attrs.get('halign'), attrs.get('valign'))
3621	# Set calculated attributes.
3622	self.attributes['colcount'] = len(self.columns)
3623	self.build_colspecs()
3624	self.parse_rows(text)
3625	# The 'rowcount' attribute is used by the experimental LaTeX backend.
3626	self.attributes['rowcount'] = str(len(self.rows))
3627	# Generate headrows, footrows, bodyrows.
3628	# Headrow, footrow and bodyrow data replaces same named attributes in
3629	# the table markup template. In order to ensure this data does not get
3630	# a second attribute substitution (which would interfere with any
3631	# already substituted inline passthroughs) unique placeholders are used
3632	# (the tab character does not appear elsewhere since it is expanded on
3633	# input) which are replaced after template attribute substitution.
3634	headrows = footrows = bodyrows = None
3635	if self.rows and 'header' in self.parameters.options:
3636	headrows = self.subs_rows(self.rows[0:1],'header')
3637	self.attributes['headrows'] = '\x07headrows\x07'
3638	self.rows = self.rows[1:]
3639	if self.rows and 'footer' in self.parameters.options:
3640	footrows = self.subs_rows( self.rows[-1:], 'footer')
3641	self.attributes['footrows'] = '\x07footrows\x07'
3642	self.rows = self.rows[:-1]
3643	if self.rows:
3644	bodyrows = self.subs_rows(self.rows)
3645	self.attributes['bodyrows'] = '\x07bodyrows\x07'
3646	table = subs_attrs(config.sections[self.parameters.template],
3647	self.attributes)
3648	table = writer.newline.join(table)
3649	# Before we finish replace the table head, foot and body place holders
3650	# with the real data.
3651	if headrows:
3652	table = table.replace('\x07headrows\x07', headrows, 1)
3653	if footrows:
3654	table = table.replace('\x07footrows\x07', footrows, 1)
3655	if bodyrows:
3656	table = table.replace('\x07bodyrows\x07', bodyrows, 1)
3657	writer.write(table,trace='table')
3658	self.pop_blockname()
3659
3660	class Tables(AbstractBlocks):
3661	"""List of tables."""
3662	BLOCK_TYPE = Table
3663	PREFIX = 'tabledef-'
3664	TAGS = ('colspec', 'headrow','footrow','bodyrow',
3665	'headdata','footdata', 'bodydata','paragraph')
3666	def __init__(self):
3667	AbstractBlocks.__init__(self)
3668	# Table tags dictionary. Each entry is a tags dictionary.
3669	self.tags={}
3670	def load(self,sections):
3671	AbstractBlocks.load(self,sections)
3672	self.load_tags(sections)
3673	def load_tags(self,sections):
3674	"""
3675	Load tabletags-* conf file sections to self.tags.
3676	"""
3677	for section in sections.keys():
3678	mo = re.match(r'^tabletags-(?P<name>\w+)$',section)
3679	if mo:
3680	name = mo.group('name')
3681	if name in self.tags:
3682	d = self.tags[name]
3683	else:
3684	d = AttrDict()
3685	parse_entries(sections.get(section,()),d)
3686	for k in d.keys():
3687	if k not in self.TAGS:
3688	message.warning('[%s] contains illegal table tag: %s' %
3689	(section,k))
3690	self.tags[name] = d
3691	def validate(self):
3692	AbstractBlocks.validate(self)
3693	# Check we have a default table definition,
3694	for i in range(len(self.blocks)):
3695	if self.blocks[i].defname == 'tabledef-default':
3696	default = self.blocks[i]
3697	break
3698	else:
3699	raise EAsciiDoc,'missing section: [tabledef-default]'
3700	# Propagate defaults to unspecified table parameters.
3701	for b in self.blocks:
3702	if b is not default:
3703	if b.format is None: b.format = default.format
3704	if b.template is None: b.template = default.template
3705	# Check tags and propagate default tags.
3706	if not 'default' in self.tags:
3707	raise EAsciiDoc,'missing section: [tabletags-default]'
3708	default = self.tags['default']
3709	for tag in ('bodyrow','bodydata','paragraph'): # Mandatory default tags.
3710	if tag not in default:
3711	raise EAsciiDoc,'missing [tabletags-default] entry: %s' % tag
3712	for t in self.tags.values():
3713	if t is not default:
3714	if t.colspec is None: t.colspec = default.colspec
3715	if t.headrow is None: t.headrow = default.headrow
3716	if t.footrow is None: t.footrow = default.footrow
3717	if t.bodyrow is None: t.bodyrow = default.bodyrow
3718	if t.headdata is None: t.headdata = default.headdata
3719	if t.footdata is None: t.footdata = default.footdata
3720	if t.bodydata is None: t.bodydata = default.bodydata
3721	if t.paragraph is None: t.paragraph = default.paragraph
3722	# Use body tags if header and footer tags are not specified.
3723	for t in self.tags.values():
3724	if not t.headrow: t.headrow = t.bodyrow
3725	if not t.footrow: t.footrow = t.bodyrow
3726	if not t.headdata: t.headdata = t.bodydata
3727	if not t.footdata: t.footdata = t.bodydata
3728	# Check table definitions are valid.
3729	for b in self.blocks:
3730	b.validate()
3731	def dump(self):
3732	AbstractBlocks.dump(self)
3733	for k,v in self.tags.items():
3734	dump_section('tabletags-'+k, v)
3735
3736	class Macros:
3737	# Default system macro syntax.
3738	SYS_RE = r'(?u)^(?P<name>[\\]?\w(\w\|-)?)::(?P<target>\S?)' + \
3739	r'(\[(?P<attrlist>.*?)\])$'
3740	def __init__(self):
3741	self.macros = [] # List of Macros.
3742	self.current = None # The last matched block macro.
3743	self.passthroughs = []
3744	# Initialize default system macro.
3745	m = Macro()
3746	m.pattern = self.SYS_RE
3747	m.prefix = '+'
3748	m.reo = re.compile(m.pattern)
3749	self.macros.append(m)
3750	def load(self,entries):
3751	for entry in entries:
3752	m = Macro()
3753	m.load(entry)
3754	if m.name is None:
3755	# Delete undefined macro.
3756	for i,m2 in enumerate(self.macros):
3757	if m2.pattern == m.pattern:
3758	del self.macros[i]
3759	break
3760	else:
3761	message.warning('unable to delete missing macro: %s' % m.pattern)
3762	else:
3763	# Check for duplicates.
3764	for m2 in self.macros:
3765	if m2.pattern == m.pattern:
3766	message.verbose('macro redefinition: %s%s' % (m.prefix,m.name))
3767	break
3768	else:
3769	self.macros.append(m)
3770	def dump(self):
3771	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
3772	write('[macros]')
3773	# Dump all macros except the first (built-in system) macro.
3774	for m in self.macros[1:]:
3775	# Escape = in pattern.
3776	macro = '%s=%s%s' % (m.pattern.replace('=',r'\='), m.prefix, m.name)
3777	if m.subslist is not None:
3778	macro += '[' + ','.join(m.subslist) + ']'
3779	write(macro)
3780	write('')
3781	def validate(self):
3782	# Check all named sections exist.
3783	if config.verbose:
3784	for m in self.macros:
3785	if m.name and m.prefix != '+':
3786	m.section_name()
3787	def subs(self,text,prefix='',callouts=False):
3788	# If callouts is True then only callout macros are processed, if False
3789	# then all non-callout macros are processed.
3790	result = text
3791	for m in self.macros:
3792	if m.prefix == prefix:
3793	if callouts ^ (m.name != 'callout'):
3794	result = m.subs(result)
3795	return result
3796	def isnext(self):
3797	"""Return matching macro if block macro is next on reader."""
3798	reader.skip_blank_lines()
3799	line = reader.read_next()
3800	if line:
3801	for m in self.macros:
3802	if m.prefix == '#':
3803	if m.reo.match(line):
3804	self.current = m
3805	return m
3806	return False
3807	def match(self,prefix,name,text):
3808	"""Return re match object matching 'text' with macro type 'prefix',
3809	macro name 'name'."""
3810	for m in self.macros:
3811	if m.prefix == prefix:
3812	mo = m.reo.match(text)
3813	if mo:
3814	if m.name == name:
3815	return mo
3816	if re.match(name, mo.group('name')):
3817	return mo
3818	return None
3819	def extract_passthroughs(self,text,prefix=''):
3820	""" Extract the passthrough text and replace with temporary
3821	placeholders."""
3822	self.passthroughs = []
3823	for m in self.macros:
3824	if m.has_passthrough() and m.prefix == prefix:
3825	text = m.subs_passthroughs(text, self.passthroughs)
3826	return text
3827	def restore_passthroughs(self,text):
3828	""" Replace passthough placeholders with the original passthrough
3829	text."""
3830	for i,v in enumerate(self.passthroughs):
3831	text = text.replace('\x07'+str(i)+'\x07', self.passthroughs[i])
3832	return text
3833
3834	class Macro:
3835	def __init__(self):
3836	self.pattern = None # Matching regular expression.
3837	self.name = '' # Conf file macro name (None if implicit).
3838	self.prefix = '' # '' if inline, '+' if system, '#' if block.
3839	self.reo = None # Compiled pattern re object.
3840	self.subslist = [] # Default subs for macros passtext group.
3841	def has_passthrough(self):
3842	return self.pattern.find(r'(?P<passtext>') >= 0
3843	def section_name(self,name=None):
3844	"""Return macro markup template section name based on macro name and
3845	prefix. Return None section not found."""
3846	assert self.prefix != '+'
3847	if not name:
3848	assert self.name
3849	name = self.name
3850	if self.prefix == '#':
3851	suffix = '-blockmacro'
3852	else:
3853	suffix = '-inlinemacro'
3854	if name+suffix in config.sections:
3855	return name+suffix
3856	else:
3857	message.warning('missing macro section: [%s]' % (name+suffix))
3858	return None
3859	def load(self,entry):
3860	e = parse_entry(entry)
3861	if e is None:
3862	# Only the macro pattern was specified, mark for deletion.
3863	self.name = None
3864	self.pattern = entry
3865	return
3866	if not is_re(e[0]):
3867	raise EAsciiDoc,'illegal macro regular expression: %s' % e[0]
3868	pattern, name = e
3869	if name and name[0] in ('+','#'):
3870	prefix, name = name[0], name[1:]
3871	else:
3872	prefix = ''
3873	# Parse passthrough subslist.
3874	mo = re.match(r'^(?P<name>[^[])(\[(?P<subslist>.)\])?$', name)
3875	name = mo.group('name')
3876	if name and not is_name(name):
3877	raise EAsciiDoc,'illegal section name in macro entry: %s' % entry
3878	subslist = mo.group('subslist')
3879	if subslist is not None:
3880	# Parse and validate passthrough subs.
3881	subslist = parse_options(subslist, SUBS_OPTIONS,
3882	'illegal subs in macro entry: %s' % entry)
3883	self.pattern = pattern
3884	self.reo = re.compile(pattern)
3885	self.prefix = prefix
3886	self.name = name
3887	self.subslist = subslist or []
3888
3889	def subs(self,text):
3890	def subs_func(mo):
3891	"""Function called to perform macro substitution.
3892	Uses matched macro regular expression object and returns string
3893	containing the substituted macro body."""
3894	# Check if macro reference is escaped.
3895	if mo.group()[0] == '\\':
3896	return mo.group()[1:] # Strip leading backslash.
3897	d = mo.groupdict()
3898	# Delete groups that didn't participate in match.
3899	for k,v in d.items():
3900	if v is None: del d[k]
3901	if self.name:
3902	name = self.name
3903	else:
3904	if not 'name' in d:
3905	message.warning('missing macro name group: %s' % mo.re.pattern)
3906	return ''
3907	name = d['name']
3908	section_name = self.section_name(name)
3909	if not section_name:
3910	return ''
3911	# If we're dealing with a block macro get optional block ID and
3912	# block title.
3913	if self.prefix == '#' and self.name != 'comment':
3914	AttributeList.consume(d)
3915	BlockTitle.consume(d)
3916	# Parse macro attributes.
3917	if 'attrlist' in d:
3918	if d['attrlist'] in (None,''):
3919	del d['attrlist']
3920	else:
3921	if self.prefix == '':
3922	# Unescape ] characters in inline macros.
3923	d['attrlist'] = d['attrlist'].replace('\\]',']')
3924	parse_attributes(d['attrlist'],d)
3925	# Generate option attributes.
3926	if 'options' in d:
3927	options = parse_options(d['options'], (),
3928	'%s: illegal option name' % name)
3929	for option in options:
3930	d[option+'-option'] = ''
3931	# Substitute single quoted attribute values in block macros.
3932	if self.prefix == '#':
3933	AttributeList.subs(d)
3934	if name == 'callout':
3935	listindex =int(d['index'])
3936	d['coid'] = calloutmap.add(listindex)
3937	# The alt attribute is the first image macro positional attribute.
3938	if name == 'image' and '1' in d:
3939	d['alt'] = d['1']
3940	# Unescape special characters in LaTeX target file names.
3941	if document.backend == 'latex' and 'target' in d and d['target']:
3942	if not '0' in d:
3943	d['0'] = d['target']
3944	d['target']= config.subs_specialchars_reverse(d['target'])
3945	# BUG: We've already done attribute substitution on the macro which
3946	# means that any escaped attribute references are now unescaped and
3947	# will be substituted by config.subs_section() below. As a partial
3948	# fix have withheld {0} from substitution but this kludge doesn't
3949	# fix it for other attributes containing unescaped references.
3950	# Passthrough macros don't have this problem.
3951	a0 = d.get('0')
3952	if a0:
3953	d['0'] = chr(0) # Replace temporarily with unused character.
3954	body = config.subs_section(section_name,d)
3955	if len(body) == 0:
3956	result = ''
3957	elif len(body) == 1:
3958	result = body[0]
3959	else:
3960	if self.prefix == '#':
3961	result = writer.newline.join(body)
3962	else:
3963	# Internally processed inline macros use UNIX line
3964	# separator.
3965	result = '\n'.join(body)
3966	if a0:
3967	result = result.replace(chr(0), a0)
3968	return result
3969
3970	return self.reo.sub(subs_func, text)
3971
3972	def translate(self):
3973	""" Block macro translation."""
3974	assert self.prefix == '#'
3975	s = reader.read()
3976	before = s
3977	if self.has_passthrough():
3978	s = macros.extract_passthroughs(s,'#')
3979	s = subs_attrs(s)
3980	if s:
3981	s = self.subs(s)
3982	if self.has_passthrough():
3983	s = macros.restore_passthroughs(s)
3984	if s:
3985	trace('macro block',before,s)
3986	writer.write(s)
3987
3988	def subs_passthroughs(self, text, passthroughs):
3989	""" Replace macro attribute lists in text with placeholders.
3990	Substitute and append the passthrough attribute lists to the
3991	passthroughs list."""
3992	def subs_func(mo):
3993	"""Function called to perform inline macro substitution.
3994	Uses matched macro regular expression object and returns string
3995	containing the substituted macro body."""
3996	# Don't process escaped macro references.
3997	if mo.group()[0] == '\\':
3998	return mo.group()
3999	d = mo.groupdict()
4000	if not 'passtext' in d:
4001	message.warning('passthrough macro %s: missing passtext group' %
4002	d.get('name',''))
4003	return mo.group()
4004	passtext = d['passtext']
4005	if re.search('\x07\\d+\x07', passtext):
4006	message.warning('nested inline passthrough')
4007	return mo.group()
4008	if d.get('subslist'):
4009	if d['subslist'].startswith(':'):
4010	message.error('block macro cannot occur here: %s' % mo.group(),
4011	halt=True)
4012	subslist = parse_options(d['subslist'], SUBS_OPTIONS,
4013	'illegal passthrough macro subs option')
4014	else:
4015	subslist = self.subslist
4016	passtext = Lex.subs_1(passtext,subslist)
4017	if passtext is None: passtext = ''
4018	if self.prefix == '':
4019	# Unescape ] characters in inline macros.
4020	passtext = passtext.replace('\\]',']')
4021	passthroughs.append(passtext)
4022	# Tabs guarantee the placeholders are unambiguous.
4023	result = (
4024	text[mo.start():mo.start('passtext')] +
4025	'\x07' + str(len(passthroughs)-1) + '\x07' +
4026	text[mo.end('passtext'):mo.end()]
4027	)
4028	return result
4029
4030	return self.reo.sub(subs_func, text)
4031
4032
4033	class CalloutMap:
4034	def __init__(self):
4035	self.comap = {} # key = list index, value = callouts list.
4036	self.calloutindex = 0 # Current callout index number.
4037	self.listnumber = 1 # Current callout list number.
4038	def listclose(self):
4039	# Called when callout list is closed.
4040	self.listnumber += 1
4041	self.calloutindex = 0
4042	self.comap = {}
4043	def add(self,listindex):
4044	# Add next callout index to listindex map entry. Return the callout id.
4045	self.calloutindex += 1
4046	# Append the coindex to a list in the comap dictionary.
4047	if not listindex in self.comap:
4048	self.comap[listindex] = [self.calloutindex]
4049	else:
4050	self.comap[listindex].append(self.calloutindex)
4051	return self.calloutid(self.listnumber, self.calloutindex)
4052	@staticmethod
4053	def calloutid(listnumber,calloutindex):
4054	return 'CO%d-%d' % (listnumber,calloutindex)
4055	def calloutids(self,listindex):
4056	# Retieve list of callout indexes that refer to listindex.
4057	if listindex in self.comap:
4058	result = ''
4059	for coindex in self.comap[listindex]:
4060	result += ' ' + self.calloutid(self.listnumber,coindex)
4061	return result.strip()
4062	else:
4063	message.warning('no callouts refer to list item '+str(listindex))
4064	return ''
4065	def validate(self,maxlistindex):
4066	# Check that all list indexes referenced by callouts exist.
4067	for listindex in self.comap.keys():
4068	if listindex > maxlistindex:
4069	message.warning('callout refers to non-existent list item '
4070	+ str(listindex))
4071
4072	#---------------------------------------------------------------------------
4073	# Input stream Reader and output stream writer classes.
4074	#---------------------------------------------------------------------------
4075
4076	UTF8_BOM = '\xef\xbb\xbf'
4077
4078	class Reader1:
4079	"""Line oriented AsciiDoc input file reader. Processes include and
4080	conditional inclusion system macros. Tabs are expanded and lines are right
4081	trimmed."""
4082	# This class is not used directly, use Reader class instead.
4083	READ_BUFFER_MIN = 10 # Read buffer low level.
4084	def __init__(self):
4085	self.f = None # Input file object.
4086	self.fname = None # Input file name.
4087	self.next = [] # Read ahead buffer containing
4088	# [filename,linenumber,linetext] lists.
4089	self.cursor = None # Last read() [filename,linenumber,linetext].
4090	self.tabsize = 8 # Tab expansion number of spaces.
4091	self.parent = None # Included reader's parent reader.
4092	self._lineno = 0 # The last line read from file object f.
4093	self.current_depth = 0 # Current include depth.
4094	self.max_depth = 10 # Initial maxiumum allowed include depth.
4095	self.bom = None # Byte order mark (BOM).
4096	self.infile = None # Saved document 'infile' attribute.
4097	self.indir = None # Saved document 'indir' attribute.
4098	def open(self,fname):
4099	self.fname = fname
4100	message.verbose('reading: '+fname)
4101	if fname == '<stdin>':
4102	self.f = sys.stdin
4103	self.infile = None
4104	self.indir = None
4105	else:
4106	self.f = open(fname,'rb')
4107	self.infile = fname
4108	self.indir = os.path.dirname(fname)
4109	document.attributes['infile'] = self.infile
4110	document.attributes['indir'] = self.indir
4111	self._lineno = 0 # The last line read from file object f.
4112	self.next = []
4113	# Prefill buffer by reading the first line and then pushing it back.
4114	if Reader1.read(self):
4115	if self.cursor[2].startswith(UTF8_BOM):
4116	self.cursor[2] = self.cursor[2][len(UTF8_BOM):]
4117	self.bom = UTF8_BOM
4118	self.unread(self.cursor)
4119	self.cursor = None
4120	def closefile(self):
4121	"""Used by class methods to close nested include files."""
4122	self.f.close()
4123	self.next = []
4124	def close(self):
4125	self.closefile()
4126	self.__init__()
4127	def read(self, skip=False):
4128	"""Read next line. Return None if EOF. Expand tabs. Strip trailing
4129	white space. Maintain self.next read ahead buffer. If skip=True then
4130	conditional exclusion is active (ifdef and ifndef macros)."""
4131	# Top up buffer.
4132	if len(self.next) <= self.READ_BUFFER_MIN:
4133	s = self.f.readline()
4134	if s:
4135	self._lineno = self._lineno + 1
4136	while s:
4137	if self.tabsize != 0:
4138	s = s.expandtabs(self.tabsize)
4139	s = s.rstrip()
4140	self.next.append([self.fname,self._lineno,s])
4141	if len(self.next) > self.READ_BUFFER_MIN:
4142	break
4143	s = self.f.readline()
4144	if s:
4145	self._lineno = self._lineno + 1
4146	# Return first (oldest) buffer entry.
4147	if len(self.next) > 0:
4148	self.cursor = self.next[0]
4149	del self.next[0]
4150	result = self.cursor[2]
4151	# Check for include macro.
4152	mo = macros.match('+',r'^include[1]?$',result)
4153	if mo and not skip:
4154	# Parse include macro attributes.
4155	attrs = {}
4156	parse_attributes(mo.group('attrlist'),attrs)
4157	warnings = attrs.get('warnings', True)
4158	# Don't process include macro once the maximum depth is reached.
4159	if self.current_depth >= self.max_depth:
4160	message.warning('maximum include depth exceeded')
4161	return result
4162	# Perform attribute substitution on include macro file name.
4163	fname = subs_attrs(mo.group('target'))
4164	if not fname:
4165	return Reader1.read(self) # Return next input line.
4166	if self.fname != '<stdin>':
4167	fname = os.path.expandvars(os.path.expanduser(fname))
4168	fname = safe_filename(fname, os.path.dirname(self.fname))
4169	if not fname:
4170	return Reader1.read(self) # Return next input line.
4171	if not os.path.isfile(fname):
4172	if warnings:
4173	message.warning('include file not found: %s' % fname)
4174	return Reader1.read(self) # Return next input line.
4175	if mo.group('name') == 'include1':
4176	if not config.dumping:
4177	if fname not in config.include1:
4178	message.verbose('include1: ' + fname, linenos=False)
4179	# Store the include file in memory for later
4180	# retrieval by the {include1:} system attribute.
4181	f = open(fname)
4182	try:
4183	config.include1[fname] = [
4184	s.rstrip() for s in f]
4185	finally:
4186	f.close()
4187	return '{include1:%s}' % fname
4188	else:
4189	# This is a configuration dump, just pass the macro
4190	# call through.
4191	return result
4192	# Clone self and set as parent (self assumes the role of child).
4193	parent = Reader1()
4194	assign(parent,self)
4195	self.parent = parent
4196	# Set attributes in child.
4197	if 'tabsize' in attrs:
4198	try:
4199	val = int(attrs['tabsize'])
4200	if not val >= 0:
4201	raise ValueError, 'not >= 0'
4202	self.tabsize = val
4203	except ValueError:
4204	raise EAsciiDoc, 'illegal include macro tabsize argument'
4205	else:
4206	self.tabsize = config.tabsize
4207	if 'depth' in attrs:
4208	try:
4209	val = int(attrs['depth'])
4210	if not val >= 1:
4211	raise ValueError, 'not >= 1'
4212	self.max_depth = self.current_depth + val
4213	except ValueError:
4214	raise EAsciiDoc, "include macro: illegal 'depth' argument"
4215	# Process included file.
4216	message.verbose('include: ' + fname, linenos=False)
4217	self.open(fname)
4218	self.current_depth = self.current_depth + 1
4219	result = Reader1.read(self)
4220	else:
4221	if not Reader1.eof(self):
4222	result = Reader1.read(self)
4223	else:
4224	result = None
4225	return result
4226	def eof(self):
4227	"""Returns True if all lines have been read."""
4228	if len(self.next) == 0:
4229	# End of current file.
4230	if self.parent:
4231	self.closefile()
4232	assign(self,self.parent) # Restore parent reader.
4233	document.attributes['infile'] = self.infile
4234	document.attributes['indir'] = self.indir
4235	return Reader1.eof(self)
4236	else:
4237	return True
4238	else:
4239	return False
4240	def read_next(self):
4241	"""Like read() but does not advance file pointer."""
4242	if Reader1.eof(self):
4243	return None
4244	else:
4245	return self.next[0][2]
4246	def unread(self,cursor):
4247	"""Push the line (filename,linenumber,linetext) tuple back into the read
4248	buffer. Note that it's up to the caller to restore the previous
4249	cursor."""
4250	assert cursor
4251	self.next.insert(0,cursor)
4252
4253	class Reader(Reader1):
4254	""" Wraps (well, sought of) Reader1 class and implements conditional text
4255	inclusion."""
4256	def __init__(self):
4257	Reader1.__init__(self)
4258	self.depth = 0 # if nesting depth.
4259	self.skip = False # true if we're skipping ifdef...endif.
4260	self.skipname = '' # Name of current endif macro target.
4261	self.skipto = -1 # The depth at which skipping is reenabled.
4262	def read_super(self):
4263	result = Reader1.read(self,self.skip)
4264	if result is None and self.skip:
4265	raise EAsciiDoc,'missing endif::%s[]' % self.skipname
4266	return result
4267	def read(self):
4268	result = self.read_super()
4269	if result is None:
4270	return None
4271	while self.skip:
4272	mo = macros.match('+',r'ifdef\|ifndef\|ifeval\|endif',result)
4273	if mo:
4274	name = mo.group('name')
4275	target = mo.group('target')
4276	attrlist = mo.group('attrlist')
4277	if name == 'endif':
4278	self.depth -= 1
4279	if self.depth < 0:
4280	raise EAsciiDoc,'mismatched macro: %s' % result
4281	if self.depth == self.skipto:
4282	self.skip = False
4283	if target and self.skipname != target:
4284	raise EAsciiDoc,'mismatched macro: %s' % result
4285	else:
4286	if name in ('ifdef','ifndef'):
4287	if not target:
4288	raise EAsciiDoc,'missing macro target: %s' % result
4289	if not attrlist:
4290	self.depth += 1
4291	elif name == 'ifeval':
4292	if not attrlist:
4293	raise EAsciiDoc,'missing ifeval condition: %s' % result
4294	self.depth += 1
4295	result = self.read_super()
4296	if result is None:
4297	return None
4298	mo = macros.match('+',r'ifdef\|ifndef\|ifeval\|endif',result)
4299	if mo:
4300	name = mo.group('name')
4301	target = mo.group('target')
4302	attrlist = mo.group('attrlist')
4303	if name == 'endif':
4304	self.depth = self.depth-1
4305	else:
4306	if not target and name in ('ifdef','ifndef'):
4307	raise EAsciiDoc,'missing macro target: %s' % result
4308	defined = is_attr_defined(target, document.attributes)
4309	if name == 'ifdef':
4310	if attrlist:
4311	if defined: return attrlist
4312	else:
4313	self.skip = not defined
4314	elif name == 'ifndef':
4315	if attrlist:
4316	if not defined: return attrlist
4317	else:
4318	self.skip = defined
4319	elif name == 'ifeval':
4320	if safe():
4321	message.unsafe('ifeval invalid')
4322	raise EAsciiDoc,'ifeval invalid safe document'
4323	if not attrlist:
4324	raise EAsciiDoc,'missing ifeval condition: %s' % result
4325	cond = False
4326	attrlist = subs_attrs(attrlist)
4327	if attrlist:
4328	try:
4329	cond = eval(attrlist)
4330	except Exception,e:
4331	raise EAsciiDoc,'error evaluating ifeval condition: %s: %s' % (result, str(e))
4332	message.verbose('ifeval: %s: %r' % (attrlist, cond))
4333	self.skip = not cond
4334	if not attrlist or name == 'ifeval':
4335	if self.skip:
4336	self.skipto = self.depth
4337	self.skipname = target
4338	self.depth = self.depth+1
4339	result = self.read()
4340	if result:
4341	# Expand executable block macros.
4342	mo = macros.match('+',r'eval\|sys\|sys2',result)
4343	if mo:
4344	action = mo.group('name')
4345	cmd = mo.group('attrlist')
4346	result = system(action, cmd, is_macro=True)
4347	self.cursor[2] = result # So we don't re-evaluate.
4348	if result:
4349	# Unescape escaped system macros.
4350	if macros.match('+',r'\\eval\|\\sys\|\\sys2\|\\ifdef\|\\ifndef\|\\endif\|\\include\|\\include1',result):
4351	result = result[1:]
4352	return result
4353	def eof(self):
4354	return self.read_next() is None
4355	def read_next(self):
4356	save_cursor = self.cursor
4357	result = self.read()
4358	if result is not None:
4359	self.unread(self.cursor)
4360	self.cursor = save_cursor
4361	return result
4362	def read_lines(self,count=1):
4363	"""Return tuple containing count lines."""
4364	result = []
4365	i = 0
4366	while i < count and not self.eof():
4367	result.append(self.read())
4368	return tuple(result)
4369	def read_ahead(self,count=1):
4370	"""Same as read_lines() but does not advance the file pointer."""
4371	result = []
4372	putback = []
4373	save_cursor = self.cursor
4374	try:
4375	i = 0
4376	while i < count and not self.eof():
4377	result.append(self.read())
4378	putback.append(self.cursor)
4379	i = i+1
4380	while putback:
4381	self.unread(putback.pop())
4382	finally:
4383	self.cursor = save_cursor
4384	return tuple(result)
4385	def skip_blank_lines(self):
4386	reader.read_until(r'\s*\S+')
4387	def read_until(self,terminators,same_file=False):
4388	"""Like read() but reads lines up to (but not including) the first line
4389	that matches the terminator regular expression, regular expression
4390	object or list of regular expression objects. If same_file is True then
4391	the terminating pattern must occur in the file the was being read when
4392	the routine was called."""
4393	if same_file:
4394	fname = self.cursor[0]
4395	result = []
4396	if not isinstance(terminators,list):
4397	if isinstance(terminators,basestring):
4398	terminators = [re.compile(terminators)]
4399	else:
4400	terminators = [terminators]
4401	while not self.eof():
4402	save_cursor = self.cursor
4403	s = self.read()
4404	if not same_file or fname == self.cursor[0]:
4405	for reo in terminators:
4406	if reo.match(s):
4407	self.unread(self.cursor)
4408	self.cursor = save_cursor
4409	return tuple(result)
4410	result.append(s)
4411	return tuple(result)
4412
4413	class Writer:
4414	"""Writes lines to output file."""
4415	def __init__(self):
4416	self.newline = '\r\n' # End of line terminator.
4417	self.f = None # Output file object.
4418	self.fname = None # Output file name.
4419	self.lines_out = 0 # Number of lines written.
4420	self.skip_blank_lines = False # If True don't output blank lines.
4421	def open(self,fname,bom=None):
4422	'''
4423	bom is optional byte order mark.
4424	http://en.wikipedia.org/wiki/Byte-order_mark
4425	'''
4426	self.fname = fname
4427	if fname == '<stdout>':
4428	self.f = sys.stdout
4429	else:
4430	self.f = open(fname,'wb+')
4431	message.verbose('writing: '+writer.fname,False)
4432	if bom:
4433	self.f.write(bom)
4434	self.lines_out = 0
4435	def close(self):
4436	if self.fname != '<stdout>':
4437	self.f.close()
4438	def write_line(self, line=None):
4439	if not (self.skip_blank_lines and (not line or not line.strip())):
4440	self.f.write((line or '') + self.newline)
4441	self.lines_out = self.lines_out + 1
4442	def write(self,args,*kwargs):
4443	"""Iterates arguments, writes tuple and list arguments one line per
4444	element, else writes argument as single line. If no arguments writes
4445	blank line. If argument is None nothing is written. self.newline is
4446	appended to each line."""
4447	if 'trace' in kwargs and len(args) > 0:
4448	trace(kwargs['trace'],args[0])
4449	if len(args) == 0:
4450	self.write_line()
4451	self.lines_out = self.lines_out + 1
4452	else:
4453	for arg in args:
4454	if is_array(arg):
4455	for s in arg:
4456	self.write_line(s)
4457	elif arg is not None:
4458	self.write_line(arg)
4459	def write_tag(self,tag,content,subs=None,d=None,**kwargs):
4460	"""Write content enveloped by tag.
4461	Substitutions specified in the 'subs' list are perform on the
4462	'content'."""
4463	if subs is None:
4464	subs = config.subsnormal
4465	stag,etag = subs_tag(tag,d)
4466	content = Lex.subs(content,subs)
4467	if 'trace' in kwargs:
4468	trace(kwargs['trace'],[stag]+content+[etag])
4469	if stag:
4470	self.write(stag)
4471	if content:
4472	self.write(content)
4473	if etag:
4474	self.write(etag)
4475
4476	#---------------------------------------------------------------------------
4477	# Configuration file processing.
4478	#---------------------------------------------------------------------------
4479	def _subs_specialwords(mo):
4480	"""Special word substitution function called by
4481	Config.subs_specialwords()."""
4482	word = mo.re.pattern # The special word.
4483	template = config.specialwords[word] # The corresponding markup template.
4484	if not template in config.sections:
4485	raise EAsciiDoc,'missing special word template [%s]' % template
4486	if mo.group()[0] == '\\':
4487	return mo.group()[1:] # Return escaped word.
4488	args = {}
4489	args['words'] = mo.group() # The full match string is argument 'words'.
4490	args.update(mo.groupdict()) # Add other named match groups to the arguments.
4491	# Delete groups that didn't participate in match.
4492	for k,v in args.items():
4493	if v is None: del args[k]
4494	lines = subs_attrs(config.sections[template],args)
4495	if len(lines) == 0:
4496	result = ''
4497	elif len(lines) == 1:
4498	result = lines[0]
4499	else:
4500	result = writer.newline.join(lines)
4501	return result
4502
4503	class Config:
4504	"""Methods to process configuration files."""
4505	# Non-template section name regexp's.
4506	ENTRIES_SECTIONS= ('tags','miscellaneous','attributes','specialcharacters',
4507	'specialwords','macros','replacements','quotes','titles',
4508	r'paradef-.+',r'listdef-.+',r'blockdef-.+',r'tabledef-.+',
4509	r'tabletags-.+',r'listtags-.+','replacements[23]',
4510	r'old_tabledef-.+')
4511	def __init__(self):
4512	self.sections = OrderedDict() # Keyed by section name containing
4513	# lists of section lines.
4514	# Command-line options.
4515	self.verbose = False
4516	self.header_footer = True # -s, --no-header-footer option.
4517	# [miscellaneous] section.
4518	self.tabsize = 8
4519	self.textwidth = 70 # DEPRECATED: Old tables only.
4520	self.newline = '\r\n'
4521	self.pagewidth = None
4522	self.pageunits = None
4523	self.outfilesuffix = ''
4524	self.subsnormal = SUBS_NORMAL
4525	self.subsverbatim = SUBS_VERBATIM
4526
4527	self.tags = {} # Values contain (stag,etag) tuples.
4528	self.specialchars = {} # Values of special character substitutions.
4529	self.specialwords = {} # Name is special word pattern, value is macro.
4530	self.replacements = OrderedDict() # Key is find pattern, value is
4531	#replace pattern.
4532	self.replacements2 = OrderedDict()
4533	self.replacements3 = OrderedDict()
4534	self.specialsections = {} # Name is special section name pattern, value
4535	# is corresponding section name.
4536	self.quotes = OrderedDict() # Values contain corresponding tag name.
4537	self.fname = '' # Most recently loaded configuration file name.
4538	self.conf_attrs = {} # Attributes entries from conf files.
4539	self.cmd_attrs = {} # Attributes from command-line -a options.
4540	self.loaded = [] # Loaded conf files.
4541	self.include1 = {} # Holds include1::[] files for {include1:}.
4542	self.dumping = False # True if asciidoc -c option specified.
4543	self.filters = [] # Filter names specified by --filter option.
4544
4545	def init(self, cmd):
4546	"""
4547	Check Python version and locate the executable and configuration files
4548	directory.
4549	cmd is the asciidoc command or asciidoc.py path.
4550	"""
4551	if float(sys.version[:3]) < float(MIN_PYTHON_VERSION):
4552	message.stderr('FAILED: Python %s or better required' %
4553	MIN_PYTHON_VERSION)
4554	sys.exit(1)
4555	if not os.path.exists(cmd):
4556	message.stderr('FAILED: Missing asciidoc command: %s' % cmd)
4557	sys.exit(1)
4558	global APP_FILE
4559	APP_FILE = os.path.realpath(cmd)
4560	global APP_DIR
4561	APP_DIR = os.path.dirname(APP_FILE)
4562	global USER_DIR
4563	USER_DIR = userdir()
4564	if USER_DIR is not None:
4565	USER_DIR = os.path.join(USER_DIR,'.asciidoc')
4566	if not os.path.isdir(USER_DIR):
4567	USER_DIR = None
4568
4569	def load_file(self, fname, dir=None, include=[], exclude=[]):
4570	"""
4571	Loads sections dictionary with sections from file fname.
4572	Existing sections are overlaid.
4573	The 'include' list contains the section names to be loaded.
4574	The 'exclude' list contains section names not to be loaded.
4575	Return False if no file was found in any of the locations.
4576	"""
4577	def update_section(section):
4578	""" Update section in sections with contents. """
4579	if section and contents:
4580	if section in sections and self.entries_section(section):
4581	if ''.join(contents):
4582	# Merge entries.
4583	sections[section] += contents
4584	else:
4585	del sections[section]
4586	else:
4587	if section.startswith('+'):
4588	# Append section.
4589	if section in sections:
4590	sections[section] += contents
4591	else:
4592	sections[section] = contents
4593	else:
4594	# Replace section.
4595	sections[section] = contents
4596	if dir:
4597	fname = os.path.join(dir, fname)
4598	# Sliently skip missing configuration file.
4599	if not os.path.isfile(fname):
4600	return False
4601	# Don't load conf files twice (local and application conf files are the
4602	# same if the source file is in the application directory).
4603	if os.path.realpath(fname) in self.loaded:
4604	return True
4605	rdr = Reader() # Reader processes system macros.
4606	message.linenos = False # Disable document line numbers.
4607	rdr.open(fname)
4608	message.linenos = None
4609	self.fname = fname
4610	reo = re.compile(r'(?u)^\[(?P<section>\+?[^\W\d][\w-])\]\s$')
4611	sections = OrderedDict()
4612	section,contents = '',[]
4613	while not rdr.eof():
4614	s = rdr.read()
4615	if s and s[0] == '#': # Skip comment lines.
4616	continue
4617	if s[:2] == '\\#': # Unescape lines starting with '#'.
4618	s = s[1:]
4619	s = s.rstrip()
4620	found = reo.findall(s)
4621	if found:
4622	update_section(section) # Store previous section.
4623	section = found[0].lower()
4624	contents = []
4625	else:
4626	contents.append(s)
4627	update_section(section) # Store last section.
4628	rdr.close()
4629	if include:
4630	for s in set(sections) - set(include):
4631	del sections[s]
4632	if exclude:
4633	for s in set(sections) & set(exclude):
4634	del sections[s]
4635	attrs = {}
4636	self.load_sections(sections,attrs)
4637	if not include:
4638	# If all sections are loaded mark this file as loaded.
4639	self.loaded.append(os.path.realpath(fname))
4640	document.update_attributes(attrs) # So they are available immediately.
4641	return True
4642
4643	def load_sections(self,sections,attrs=None):
4644	"""
4645	Loads sections dictionary. Each dictionary entry contains a
4646	list of lines.
4647	Updates 'attrs' with parsed [attributes] section entries.
4648	"""
4649	# Delete trailing blank lines from sections.
4650	for k in sections.keys():
4651	for i in range(len(sections[k])-1,-1,-1):
4652	if not sections[k][i]:
4653	del sections[k][i]
4654	elif not self.entries_section(k):
4655	break
4656	# Update new sections.
4657	for k,v in sections.items():
4658	if k.startswith('+'):
4659	# Append section.
4660	k = k[1:]
4661	if k in self.sections:
4662	self.sections[k] += v
4663	else:
4664	self.sections[k] = v
4665	else:
4666	# Replace section.
4667	self.sections[k] = v
4668	self.parse_tags()
4669	# Internally [miscellaneous] section entries are just attributes.
4670	d = {}
4671	parse_entries(sections.get('miscellaneous',()), d, unquote=True,
4672	allow_name_only=True)
4673	parse_entries(sections.get('attributes',()), d, unquote=True,
4674	allow_name_only=True)
4675	update_attrs(self.conf_attrs,d)
4676	if attrs is not None:
4677	attrs.update(d)
4678	d = {}
4679	parse_entries(sections.get('titles',()),d)
4680	Title.load(d)
4681	parse_entries(sections.get('specialcharacters',()),self.specialchars,escape_delimiter=False)
4682	parse_entries(sections.get('quotes',()),self.quotes)
4683	self.parse_specialwords()
4684	self.parse_replacements()
4685	self.parse_replacements('replacements2')
4686	self.parse_replacements('replacements3')
4687	self.parse_specialsections()
4688	paragraphs.load(sections)
4689	lists.load(sections)
4690	blocks.load(sections)
4691	tables_OLD.load(sections)
4692	tables.load(sections)
4693	macros.load(sections.get('macros',()))
4694
4695	def get_load_dirs(self):
4696	"""
4697	Return list of well known paths with conf files.
4698	"""
4699	result = []
4700	if localapp():
4701	# Load from folders in asciidoc executable directory.
4702	result.append(APP_DIR)
4703	else:
4704	# Load from global configuration directory.
4705	result.append(CONF_DIR)
4706	# Load configuration files from ~/.asciidoc if it exists.
4707	if USER_DIR is not None:
4708	result.append(USER_DIR)
4709	return result
4710
4711	def find_in_dirs(self, filename, dirs=None):
4712	"""
4713	Find conf files from dirs list.
4714	Return list of found file paths.
4715	Return empty list if not found in any of the locations.
4716	"""
4717	result = []
4718	if dirs is None:
4719	dirs = self.get_load_dirs()
4720	for d in dirs:
4721	f = os.path.join(d,filename)
4722	if os.path.isfile(f):
4723	result.append(f)
4724	return result
4725
4726	def load_from_dirs(self, filename, dirs=None, include=[]):
4727	"""
4728	Load conf file from dirs list.
4729	If dirs not specified try all the well known locations.
4730	Return False if no file was sucessfully loaded.
4731	"""
4732	count = 0
4733	for f in self.find_in_dirs(filename,dirs):
4734	if self.load_file(f, include=include):
4735	count += 1
4736	return count != 0
4737
4738	def load_backend(self, dirs=None):
4739	"""
4740	Load the backend configuration files from dirs list.
4741	If dirs not specified try all the well known locations.
4742	If a <backend>.conf file was found return it's full path name,
4743	if not found return None.
4744	"""
4745	result = None
4746	if dirs is None:
4747	dirs = self.get_load_dirs()
4748	conf = document.backend + '.conf'
4749	conf2 = document.backend + '-' + document.doctype + '.conf'
4750	# First search for filter backends.
4751	for d in [os.path.join(d, 'backends', document.backend) for d in dirs]:
4752	if self.load_file(conf,d):
4753	result = os.path.join(d, conf)
4754	self.load_file(conf2,d)
4755	if not result:
4756	# Search in the normal locations.
4757	for d in dirs:
4758	if self.load_file(conf,d):
4759	result = os.path.join(d, conf)
4760	self.load_file(conf2,d)
4761	return result
4762
4763	def load_filters(self, dirs=None):
4764	"""
4765	Load filter configuration files from 'filters' directory in dirs list.
4766	If dirs not specified try all the well known locations. Suppress
4767	loading if a file named __noautoload__ is in same directory as the conf
4768	file unless the filter has been specified with the --filter
4769	command-line option (in which case it is loaded unconditionally).
4770	"""
4771	if dirs is None:
4772	dirs = self.get_load_dirs()
4773	for d in dirs:
4774	# Load filter .conf files.
4775	filtersdir = os.path.join(d,'filters')
4776	for dirpath,dirnames,filenames in os.walk(filtersdir):
4777	subdirs = dirpath[len(filtersdir):].split(os.path.sep)
4778	# True if processing a filter specified by a --filter option.
4779	filter_opt = len(subdirs) > 1 and subdirs[1] in self.filters
4780	if '__noautoload__' not in filenames or filter_opt:
4781	for f in filenames:
4782	if re.match(r'^.+\.conf$',f):
4783	self.load_file(f,dirpath)
4784
4785	def find_config_dir(self, *dirnames):
4786	"""
4787	Return path of configuration directory.
4788	Try all the well known locations.
4789	Return None if directory not found.
4790	"""
4791	for d in [os.path.join(d, *dirnames) for d in self.get_load_dirs()]:
4792	if os.path.isdir(d):
4793	return d
4794	return None
4795
4796	def set_theme_attributes(self):
4797	theme = document.attributes.get('theme')
4798	if theme and 'themedir' not in document.attributes:
4799	themedir = self.find_config_dir('themes', theme)
4800	if themedir:
4801	document.attributes['themedir'] = themedir
4802	iconsdir = os.path.join(themedir, 'icons')
4803	if 'data-uri' in document.attributes and os.path.isdir(iconsdir):
4804	document.attributes['iconsdir'] = iconsdir
4805	else:
4806	message.warning('missing theme: %s' % theme, linenos=False)
4807
4808	def load_miscellaneous(self,d):
4809	"""Set miscellaneous configuration entries from dictionary 'd'."""
4810	def set_if_int_gt_zero(name, d):
4811	if name in d:
4812	try:
4813	val = int(d[name])
4814	if not val > 0:
4815	raise ValueError, "not > 0"
4816	if val > 0:
4817	setattr(self, name, val)
4818	except ValueError:
4819	raise EAsciiDoc, 'illegal [miscellaneous] %s entry' % name
4820	set_if_int_gt_zero('tabsize', d)
4821	set_if_int_gt_zero('textwidth', d) # DEPRECATED: Old tables only.
4822
4823	if 'pagewidth' in d:
4824	try:
4825	val = float(d['pagewidth'])
4826	self.pagewidth = val
4827	except ValueError:
4828	raise EAsciiDoc, 'illegal [miscellaneous] pagewidth entry'
4829
4830	if 'pageunits' in d:
4831	self.pageunits = d['pageunits']
4832	if 'outfilesuffix' in d:
4833	self.outfilesuffix = d['outfilesuffix']
4834	if 'newline' in d:
4835	# Convert escape sequences to their character values.
4836	self.newline = literal_eval('"'+d['newline']+'"')
4837	if 'subsnormal' in d:
4838	self.subsnormal = parse_options(d['subsnormal'],SUBS_OPTIONS,
4839	'illegal [%s] %s: %s' %
4840	('miscellaneous','subsnormal',d['subsnormal']))
4841	if 'subsverbatim' in d:
4842	self.subsverbatim = parse_options(d['subsverbatim'],SUBS_OPTIONS,
4843	'illegal [%s] %s: %s' %
4844	('miscellaneous','subsverbatim',d['subsverbatim']))
4845
4846	def validate(self):
4847	"""Check the configuration for internal consistancy. Called after all
4848	configuration files have been loaded."""
4849	message.linenos = False # Disable document line numbers.
4850	# Heuristic to validate that at least one configuration file was loaded.
4851	if not self.specialchars or not self.tags or not lists:
4852	raise EAsciiDoc,'incomplete configuration files'
4853	# Check special characters are only one character long.
4854	for k in self.specialchars.keys():
4855	if len(k) != 1:
4856	raise EAsciiDoc,'[specialcharacters] ' \
4857	'must be a single character: %s' % k
4858	# Check all special words have a corresponding inline macro body.
4859	for macro in self.specialwords.values():
4860	if not is_name(macro):
4861	raise EAsciiDoc,'illegal special word name: %s' % macro
4862	if not macro in self.sections:
4863	message.warning('missing special word macro: [%s]' % macro)
4864	# Check all text quotes have a corresponding tag.
4865	for q in self.quotes.keys()[:]:
4866	tag = self.quotes[q]
4867	if not tag:
4868	del self.quotes[q] # Undefine quote.
4869	else:
4870	if tag[0] == '#':
4871	tag = tag[1:]
4872	if not tag in self.tags:
4873	message.warning('[quotes] %s missing tag definition: %s' % (q,tag))
4874	# Check all specialsections section names exist.
4875	for k,v in self.specialsections.items():
4876	if not v:
4877	del self.specialsections[k]
4878	elif not v in self.sections:
4879	message.warning('missing specialsections section: [%s]' % v)
4880	paragraphs.validate()
4881	lists.validate()
4882	blocks.validate()
4883	tables_OLD.validate()
4884	tables.validate()
4885	macros.validate()
4886	message.linenos = None
4887
4888	def entries_section(self,section_name):
4889	"""
4890	Return True if conf file section contains entries, not a markup
4891	template.
4892	"""
4893	for name in self.ENTRIES_SECTIONS:
4894	if re.match(name,section_name):
4895	return True
4896	return False
4897
4898	def dump(self):
4899	"""Dump configuration to stdout."""
4900	# Header.
4901	hdr = ''
4902	hdr = hdr + '#' + writer.newline
4903	hdr = hdr + '# Generated by AsciiDoc %s for %s %s.%s' % \
4904	(VERSION,document.backend,document.doctype,writer.newline)
4905	t = time.asctime(time.localtime(time.time()))
4906	hdr = hdr + '# %s%s' % (t,writer.newline)
4907	hdr = hdr + '#' + writer.newline
4908	sys.stdout.write(hdr)
4909	# Dump special sections.
4910	# Dump only the configuration file and command-line attributes.
4911	# [miscellanous] entries are dumped as part of the [attributes].
4912	d = {}
4913	d.update(self.conf_attrs)
4914	d.update(self.cmd_attrs)
4915	dump_section('attributes',d)
4916	Title.dump()
4917	dump_section('quotes',self.quotes)
4918	dump_section('specialcharacters',self.specialchars)
4919	d = {}
4920	for k,v in self.specialwords.items():
4921	if v in d:
4922	d[v] = '%s "%s"' % (d[v],k) # Append word list.
4923	else:
4924	d[v] = '"%s"' % k
4925	dump_section('specialwords',d)
4926	dump_section('replacements',self.replacements)
4927	dump_section('replacements2',self.replacements2)
4928	dump_section('replacements3',self.replacements3)
4929	dump_section('specialsections',self.specialsections)
4930	d = {}
4931	for k,v in self.tags.items():
4932	d[k] = '%s\|%s' % v
4933	dump_section('tags',d)
4934	paragraphs.dump()
4935	lists.dump()
4936	blocks.dump()
4937	tables_OLD.dump()
4938	tables.dump()
4939	macros.dump()
4940	# Dump remaining sections.
4941	for k in self.sections.keys():
4942	if not self.entries_section(k):
4943	sys.stdout.write('[%s]%s' % (k,writer.newline))
4944	for line in self.sections[k]:
4945	sys.stdout.write('%s%s' % (line,writer.newline))
4946	sys.stdout.write(writer.newline)
4947
4948	def subs_section(self,section,d):
4949	"""Section attribute substitution using attributes from
4950	document.attributes and 'd'. Lines containing undefinded
4951	attributes are deleted."""
4952	if section in self.sections:
4953	return subs_attrs(self.sections[section],d)
4954	else:
4955	message.warning('missing section: [%s]' % section)
4956	return ()
4957
4958	def parse_tags(self):
4959	"""Parse [tags] section entries into self.tags dictionary."""
4960	d = {}
4961	parse_entries(self.sections.get('tags',()),d)
4962	for k,v in d.items():
4963	if v is None:
4964	if k in self.tags:
4965	del self.tags[k]
4966	elif v == '':
4967	self.tags[k] = (None,None)
4968	else:
4969	mo = re.match(r'(?P<stag>.)\\|(?P<etag>.)',v)
4970	if mo:
4971	self.tags[k] = (mo.group('stag'), mo.group('etag'))
4972	else:
4973	raise EAsciiDoc,'[tag] %s value malformed' % k
4974
4975	def tag(self, name, d=None):
4976	"""Returns (starttag,endtag) tuple named name from configuration file
4977	[tags] section. Raise error if not found. If a dictionary 'd' is
4978	passed then merge with document attributes and perform attribute
4979	substitution on tags."""
4980	if not name in self.tags:
4981	raise EAsciiDoc, 'missing tag: %s' % name
4982	stag,etag = self.tags[name]
4983	if d is not None:
4984	# TODO: Should we warn if substitution drops a tag?
4985	if stag:
4986	stag = subs_attrs(stag,d)
4987	if etag:
4988	etag = subs_attrs(etag,d)
4989	if stag is None: stag = ''
4990	if etag is None: etag = ''
4991	return (stag,etag)
4992
4993	def parse_specialsections(self):
4994	"""Parse specialsections section to self.specialsections dictionary."""
4995	# TODO: This is virtually the same as parse_replacements() and should
4996	# be factored to single routine.
4997	d = {}
4998	parse_entries(self.sections.get('specialsections',()),d,unquote=True)
4999	for pat,sectname in d.items():
5000	pat = strip_quotes(pat)
5001	if not is_re(pat):
5002	raise EAsciiDoc,'[specialsections] entry ' \
5003	'is not a valid regular expression: %s' % pat
5004	if sectname is None:
5005	if pat in self.specialsections:
5006	del self.specialsections[pat]
5007	else:
5008	self.specialsections[pat] = sectname
5009
5010	def parse_replacements(self,sect='replacements'):
5011	"""Parse replacements section into self.replacements dictionary."""
5012	d = OrderedDict()
5013	parse_entries(self.sections.get(sect,()), d, unquote=True)
5014	for pat,rep in d.items():
5015	if not self.set_replacement(pat, rep, getattr(self,sect)):
5016	raise EAsciiDoc,'[%s] entry in %s is not a valid' \
5017	' regular expression: %s' % (sect,self.fname,pat)
5018
5019	@staticmethod
5020	def set_replacement(pat, rep, replacements):
5021	"""Add pattern and replacement to replacements dictionary."""
5022	pat = strip_quotes(pat)
5023	if not is_re(pat):
5024	return False
5025	if rep is None:
5026	if pat in replacements:
5027	del replacements[pat]
5028	else:
5029	replacements[pat] = strip_quotes(rep)
5030	return True
5031
5032	def subs_replacements(self,s,sect='replacements'):
5033	"""Substitute patterns from self.replacements in 's'."""
5034	result = s
5035	for pat,rep in getattr(self,sect).items():
5036	result = re.sub(pat, rep, result)
5037	return result
5038
5039	def parse_specialwords(self):
5040	"""Parse special words section into self.specialwords dictionary."""
5041	reo = re.compile(r'(?:\s\|^)(".+?"\|[^"\s]+)(?=\s\|$)')
5042	for line in self.sections.get('specialwords',()):
5043	e = parse_entry(line)
5044	if not e:
5045	raise EAsciiDoc,'[specialwords] entry in %s is malformed: %s' \
5046	% (self.fname,line)
5047	name,wordlist = e
5048	if not is_name(name):
5049	raise EAsciiDoc,'[specialwords] name in %s is illegal: %s' \
5050	% (self.fname,name)
5051	if wordlist is None:
5052	# Undefine all words associated with 'name'.
5053	for k,v in self.specialwords.items():
5054	if v == name:
5055	del self.specialwords[k]
5056	else:
5057	words = reo.findall(wordlist)
5058	for word in words:
5059	word = strip_quotes(word)
5060	if not is_re(word):
5061	raise EAsciiDoc,'[specialwords] entry in %s ' \
5062	'is not a valid regular expression: %s' \
5063	% (self.fname,word)
5064	self.specialwords[word] = name
5065
5066	def subs_specialchars(self,s):
5067	"""Perform special character substitution on string 's'."""
5068	"""It may seem like a good idea to escape special characters with a '\'
5069	character, the reason we don't is because the escape character itself
5070	then has to be escaped and this makes including code listings
5071	problematic. Use the predefined {amp},{lt},{gt} attributes instead."""
5072	result = ''
5073	for ch in s:
5074	result = result + self.specialchars.get(ch,ch)
5075	return result
5076
5077	def subs_specialchars_reverse(self,s):
5078	"""Perform reverse special character substitution on string 's'."""
5079	result = s
5080	for k,v in self.specialchars.items():
5081	result = result.replace(v, k)
5082	return result
5083
5084	def subs_specialwords(self,s):
5085	"""Search for word patterns from self.specialwords in 's' and
5086	substitute using corresponding macro."""
5087	result = s
5088	for word in self.specialwords.keys():
5089	result = re.sub(word, _subs_specialwords, result)
5090	return result
5091
5092	def expand_templates(self,entries):
5093	"""Expand any template::[] macros in a list of section entries."""
5094	result = []
5095	for line in entries:
5096	mo = macros.match('+',r'template',line)
5097	if mo:
5098	s = mo.group('attrlist')
5099	if s in self.sections:
5100	result += self.expand_templates(self.sections[s])
5101	else:
5102	message.warning('missing section: [%s]' % s)
5103	result.append(line)
5104	else:
5105	result.append(line)
5106	return result
5107
5108	def expand_all_templates(self):
5109	for k,v in self.sections.items():
5110	self.sections[k] = self.expand_templates(v)
5111
5112	def section2tags(self, section, d={}, skipstart=False, skipend=False):
5113	"""Perform attribute substitution on 'section' using document
5114	attributes plus 'd' attributes. Return tuple (stag,etag) containing
5115	pre and post \| placeholder tags. 'skipstart' and 'skipend' are
5116	used to suppress substitution."""
5117	assert section is not None
5118	if section in self.sections:
5119	body = self.sections[section]
5120	else:
5121	message.warning('missing section: [%s]' % section)
5122	body = ()
5123	# Split macro body into start and end tag lists.
5124	stag = []
5125	etag = []
5126	in_stag = True
5127	for s in body:
5128	if in_stag:
5129	mo = re.match(r'(?P<stag>.)\\|(?P<etag>.)',s)
5130	if mo:
5131	if mo.group('stag'):
5132	stag.append(mo.group('stag'))
5133	if mo.group('etag'):
5134	etag.append(mo.group('etag'))
5135	in_stag = False
5136	else:
5137	stag.append(s)
5138	else:
5139	etag.append(s)
5140	# Do attribute substitution last so {brkbar} can be used to escape \|.
5141	# But don't do attribute substitution on title -- we've already done it.
5142	title = d.get('title')
5143	if title:
5144	d['title'] = chr(0) # Replace with unused character.
5145	if not skipstart:
5146	stag = subs_attrs(stag, d)
5147	if not skipend:
5148	etag = subs_attrs(etag, d)
5149	# Put the {title} back.
5150	if title:
5151	stag = map(lambda x: x.replace(chr(0), title), stag)
5152	etag = map(lambda x: x.replace(chr(0), title), etag)
5153	d['title'] = title
5154	return (stag,etag)
5155
5156
5157	#---------------------------------------------------------------------------
5158	# Deprecated old table classes follow.
5159	# Naming convention is an _OLD name suffix.
5160	# These will be removed from future versions of AsciiDoc
5161
5162	def join_lines_OLD(lines):
5163	"""Return a list in which lines terminated with the backslash line
5164	continuation character are joined."""
5165	result = []
5166	s = ''
5167	continuation = False
5168	for line in lines:
5169	if line and line[-1] == '\\':
5170	s = s + line[:-1]
5171	continuation = True
5172	continue
5173	if continuation:
5174	result.append(s+line)
5175	s = ''
5176	continuation = False
5177	else:
5178	result.append(line)
5179	if continuation:
5180	result.append(s)
5181	return result
5182
5183	class Column_OLD:
5184	"""Table column."""
5185	def __init__(self):
5186	self.colalign = None # 'left','right','center'
5187	self.rulerwidth = None
5188	self.colwidth = None # Output width in page units.
5189
5190	class Table_OLD(AbstractBlock):
5191	COL_STOP = r"(`\|'\|\.)" # RE.
5192	ALIGNMENTS = {'`':'left', "'":'right', '.':'center'}
5193	FORMATS = ('fixed','csv','dsv')
5194	def __init__(self):
5195	AbstractBlock.__init__(self)
5196	self.CONF_ENTRIES += ('template','fillchar','format','colspec',
5197	'headrow','footrow','bodyrow','headdata',
5198	'footdata', 'bodydata')
5199	# Configuration parameters.
5200	self.fillchar=None
5201	self.format=None # 'fixed','csv','dsv'
5202	self.colspec=None
5203	self.headrow=None
5204	self.footrow=None
5205	self.bodyrow=None
5206	self.headdata=None
5207	self.footdata=None
5208	self.bodydata=None
5209	# Calculated parameters.
5210	self.underline=None # RE matching current table underline.
5211	self.isnumeric=False # True if numeric ruler.
5212	self.tablewidth=None # Optional table width scale factor.
5213	self.columns=[] # List of Columns.
5214	# Other.
5215	self.check_msg='' # Message set by previous self.validate() call.
5216	def load(self,name,entries):
5217	AbstractBlock.load(self,name,entries)
5218	"""Update table definition from section entries in 'entries'."""
5219	for k,v in entries.items():
5220	if k == 'fillchar':
5221	if v and len(v) == 1:
5222	self.fillchar = v
5223	else:
5224	raise EAsciiDoc,'malformed table fillchar: %s' % v
5225	elif k == 'format':
5226	if v in Table_OLD.FORMATS:
5227	self.format = v
5228	else:
5229	raise EAsciiDoc,'illegal table format: %s' % v
5230	elif k == 'colspec':
5231	self.colspec = v
5232	elif k == 'headrow':
5233	self.headrow = v
5234	elif k == 'footrow':
5235	self.footrow = v
5236	elif k == 'bodyrow':
5237	self.bodyrow = v
5238	elif k == 'headdata':
5239	self.headdata = v
5240	elif k == 'footdata':
5241	self.footdata = v
5242	elif k == 'bodydata':
5243	self.bodydata = v
5244	def dump(self):
5245	AbstractBlock.dump(self)
5246	write = lambda s: sys.stdout.write('%s%s' % (s,writer.newline))
5247	write('fillchar='+self.fillchar)
5248	write('format='+self.format)
5249	if self.colspec:
5250	write('colspec='+self.colspec)
5251	if self.headrow:
5252	write('headrow='+self.headrow)
5253	if self.footrow:
5254	write('footrow='+self.footrow)
5255	write('bodyrow='+self.bodyrow)
5256	if self.headdata:
5257	write('headdata='+self.headdata)
5258	if self.footdata:
5259	write('footdata='+self.footdata)
5260	write('bodydata='+self.bodydata)
5261	write('')
5262	def validate(self):
5263	AbstractBlock.validate(self)
5264	"""Check table definition and set self.check_msg if invalid else set
5265	self.check_msg to blank string."""
5266	# Check global table parameters.
5267	if config.textwidth is None:
5268	self.check_msg = 'missing [miscellaneous] textwidth entry'
5269	elif config.pagewidth is None:
5270	self.check_msg = 'missing [miscellaneous] pagewidth entry'
5271	elif config.pageunits is None:
5272	self.check_msg = 'missing [miscellaneous] pageunits entry'
5273	elif self.headrow is None:
5274	self.check_msg = 'missing headrow entry'
5275	elif self.footrow is None:
5276	self.check_msg = 'missing footrow entry'
5277	elif self.bodyrow is None:
5278	self.check_msg = 'missing bodyrow entry'
5279	elif self.headdata is None:
5280	self.check_msg = 'missing headdata entry'
5281	elif self.footdata is None:
5282	self.check_msg = 'missing footdata entry'
5283	elif self.bodydata is None:
5284	self.check_msg = 'missing bodydata entry'
5285	else:
5286	# No errors.
5287	self.check_msg = ''
5288	def isnext(self):
5289	return AbstractBlock.isnext(self)
5290	def parse_ruler(self,ruler):
5291	"""Parse ruler calculating underline and ruler column widths."""
5292	fc = re.escape(self.fillchar)
5293	# Strip and save optional tablewidth from end of ruler.
5294	mo = re.match(r'^(.*'+fc+r'+)([\d\.]+)$',ruler)
5295	if mo:
5296	ruler = mo.group(1)
5297	self.tablewidth = float(mo.group(2))
5298	self.attributes['tablewidth'] = str(float(self.tablewidth))
5299	else:
5300	self.tablewidth = None
5301	self.attributes['tablewidth'] = '100.0'
5302	# Guess whether column widths are specified numerically or not.
5303	if ruler[1] != self.fillchar:
5304	# If the first column does not start with a fillchar then numeric.
5305	self.isnumeric = True
5306	elif ruler[1:] == self.fillchar*len(ruler[1:]):
5307	# The case of one column followed by fillchars is numeric.
5308	self.isnumeric = True
5309	else:
5310	self.isnumeric = False
5311	# Underlines must be 3 or more fillchars.
5312	self.underline = r'^' + fc + r'{3,}$'
5313	splits = re.split(self.COL_STOP,ruler)[1:]
5314	# Build self.columns.
5315	for i in range(0,len(splits),2):
5316	c = Column_OLD()
5317	c.colalign = self.ALIGNMENTS[splits[i]]
5318	s = splits[i+1]
5319	if self.isnumeric:
5320	# Strip trailing fillchars.
5321	s = re.sub(fc+r'+$','',s)
5322	if s == '':
5323	c.rulerwidth = None
5324	else:
5325	try:
5326	val = int(s)
5327	if not val > 0:
5328	raise ValueError, 'not > 0'
5329	c.rulerwidth = val
5330	except ValueError:
5331	raise EAsciiDoc, 'malformed ruler: bad width'
5332	else: # Calculate column width from inter-fillchar intervals.
5333	if not re.match(r'^'+fc+r'+$',s):
5334	raise EAsciiDoc,'malformed ruler: illegal fillchars'
5335	c.rulerwidth = len(s)+1
5336	self.columns.append(c)
5337	# Fill in unspecified ruler widths.
5338	if self.isnumeric:
5339	if self.columns[0].rulerwidth is None:
5340	prevwidth = 1
5341	for c in self.columns:
5342	if c.rulerwidth is None:
5343	c.rulerwidth = prevwidth
5344	prevwidth = c.rulerwidth
5345	def build_colspecs(self):
5346	"""Generate colwidths and colspecs. This can only be done after the
5347	table arguments have been parsed since we use the table format."""
5348	self.attributes['cols'] = len(self.columns)
5349	# Calculate total ruler width.
5350	totalwidth = 0
5351	for c in self.columns:
5352	totalwidth = totalwidth + c.rulerwidth
5353	if totalwidth <= 0:
5354	raise EAsciiDoc,'zero width table'
5355	# Calculate marked up colwidths from rulerwidths.
5356	for c in self.columns:
5357	# Convert ruler width to output page width.
5358	width = float(c.rulerwidth)
5359	if self.format == 'fixed':
5360	if self.tablewidth is None:
5361	# Size proportional to ruler width.
5362	colfraction = width/config.textwidth
5363	else:
5364	# Size proportional to page width.
5365	colfraction = width/totalwidth
5366	else:
5367	# Size proportional to page width.
5368	colfraction = width/totalwidth
5369	c.colwidth = colfraction * config.pagewidth # To page units.
5370	if self.tablewidth is not None:
5371	c.colwidth = c.colwidth * self.tablewidth # Scale factor.
5372	if self.tablewidth > 1:
5373	c.colwidth = c.colwidth/100 # tablewidth is in percent.
5374	# Build colspecs.
5375	if self.colspec:
5376	cols = []
5377	i = 0
5378	for c in self.columns:
5379	i += 1
5380	self.attributes['colalign'] = c.colalign
5381	self.attributes['colwidth'] = str(int(c.colwidth))
5382	self.attributes['colnumber'] = str(i + 1)
5383	s = subs_attrs(self.colspec,self.attributes)
5384	if not s:
5385	message.warning('colspec dropped: contains undefined attribute')
5386	else:
5387	cols.append(s)
5388	self.attributes['colspecs'] = writer.newline.join(cols)
5389	def split_rows(self,rows):
5390	"""Return a two item tuple containing a list of lines up to but not
5391	including the next underline (continued lines are joined ) and the
5392	tuple of all lines after the underline."""
5393	reo = re.compile(self.underline)
5394	i = 0
5395	while not reo.match(rows[i]):
5396	i = i+1
5397	if i == 0:
5398	raise EAsciiDoc,'missing table rows'
5399	if i >= len(rows):
5400	raise EAsciiDoc,'closing [%s] underline expected' % self.defname
5401	return (join_lines_OLD(rows[:i]), rows[i+1:])
5402	def parse_rows(self, rows, rtag, dtag):
5403	"""Parse rows list using the row and data tags. Returns a substituted
5404	list of output lines."""
5405	result = []
5406	# Source rows are parsed as single block, rather than line by line, to
5407	# allow the CSV reader to handle multi-line rows.
5408	if self.format == 'fixed':
5409	rows = self.parse_fixed(rows)
5410	elif self.format == 'csv':
5411	rows = self.parse_csv(rows)
5412	elif self.format == 'dsv':
5413	rows = self.parse_dsv(rows)
5414	else:
5415	assert True,'illegal table format'
5416	# Substitute and indent all data in all rows.
5417	stag,etag = subs_tag(rtag,self.attributes)
5418	for row in rows:
5419	result.append(' '+stag)
5420	for data in self.subs_row(row,dtag):
5421	result.append(' '+data)
5422	result.append(' '+etag)
5423	return result
5424	def subs_row(self, data, dtag):
5425	"""Substitute the list of source row data elements using the data tag.
5426	Returns a substituted list of output table data items."""
5427	result = []
5428	if len(data) < len(self.columns):
5429	message.warning('fewer row data items then table columns')
5430	if len(data) > len(self.columns):
5431	message.warning('more row data items than table columns')
5432	for i in range(len(self.columns)):
5433	if i > len(data) - 1:
5434	d = '' # Fill missing column data with blanks.
5435	else:
5436	d = data[i]
5437	c = self.columns[i]
5438	self.attributes['colalign'] = c.colalign
5439	self.attributes['colwidth'] = str(int(c.colwidth))
5440	self.attributes['colnumber'] = str(i + 1)
5441	stag,etag = subs_tag(dtag,self.attributes)
5442	# Insert AsciiDoc line break (' +') where row data has newlines
5443	# ('\n'). This is really only useful when the table format is csv
5444	# and the output markup is HTML. It's also a bit dubious in that it
5445	# assumes the user has not modified the shipped line break pattern.
5446	subs = self.get_subs()[0]
5447	if 'replacements2' in subs:
5448	# Insert line breaks in cell data.
5449	d = re.sub(r'(?m)\n',r' +\n',d)
5450	d = d.split('\n') # So writer.newline is written.
5451	else:
5452	d = [d]
5453	result = result + [stag] + Lex.subs(d,subs) + [etag]
5454	return result
5455	def parse_fixed(self,rows):
5456	"""Parse the list of source table rows. Each row item in the returned
5457	list contains a list of cell data elements."""
5458	result = []
5459	for row in rows:
5460	data = []
5461	start = 0
5462	# build an encoded representation
5463	row = char_decode(row)
5464	for c in self.columns:
5465	end = start + c.rulerwidth
5466	if c is self.columns[-1]:
5467	# Text in last column can continue forever.
5468	# Use the encoded string to slice, but convert back
5469	# to plain string before further processing
5470	data.append(char_encode(row[start:]).strip())
5471	else:
5472	data.append(char_encode(row[start:end]).strip())
5473	start = end
5474	result.append(data)
5475	return result
5476	def parse_csv(self,rows):
5477	"""Parse the list of source table rows. Each row item in the returned
5478	list contains a list of cell data elements."""
5479	import StringIO
5480	import csv
5481	result = []
5482	rdr = csv.reader(StringIO.StringIO('\r\n'.join(rows)),
5483	skipinitialspace=True)
5484	try:
5485	for row in rdr:
5486	result.append(row)
5487	except Exception:
5488	raise EAsciiDoc,'csv parse error: %s' % row
5489	return result
5490	def parse_dsv(self,rows):
5491	"""Parse the list of source table rows. Each row item in the returned
5492	list contains a list of cell data elements."""
5493	separator = self.attributes.get('separator',':')
5494	separator = literal_eval('"'+separator+'"')
5495	if len(separator) != 1:
5496	raise EAsciiDoc,'malformed dsv separator: %s' % separator
5497	# TODO If separator is preceeded by an odd number of backslashes then
5498	# it is escaped and should not delimit.
5499	result = []
5500	for row in rows:
5501	# Skip blank lines
5502	if row == '': continue
5503	# Unescape escaped characters.
5504	row = literal_eval('"'+row.replace('"','\\"')+'"')
5505	data = row.split(separator)
5506	data = [s.strip() for s in data]
5507	result.append(data)
5508	return result
5509	def translate(self):
5510	message.deprecated('old tables syntax')
5511	AbstractBlock.translate(self)
5512	# Reset instance specific properties.
5513	self.underline = None
5514	self.columns = []
5515	attrs = {}
5516	BlockTitle.consume(attrs)
5517	# Add relevant globals to table substitutions.
5518	attrs['pagewidth'] = str(config.pagewidth)
5519	attrs['pageunits'] = config.pageunits
5520	# Mix in document attribute list.
5521	AttributeList.consume(attrs)
5522	# Validate overridable attributes.
5523	for k,v in attrs.items():
5524	if k == 'format':
5525	if v not in self.FORMATS:
5526	raise EAsciiDoc, 'illegal [%s] %s: %s' % (self.defname,k,v)
5527	self.format = v
5528	elif k == 'tablewidth':
5529	try:
5530	self.tablewidth = float(attrs['tablewidth'])
5531	except Exception:
5532	raise EAsciiDoc, 'illegal [%s] %s: %s' % (self.defname,k,v)
5533	self.merge_attributes(attrs)
5534	# Parse table ruler.
5535	ruler = reader.read()
5536	assert re.match(self.delimiter,ruler)
5537	self.parse_ruler(ruler)
5538	# Read the entire table.
5539	table = []
5540	while True:
5541	line = reader.read_next()
5542	# Table terminated by underline followed by a blank line or EOF.
5543	if len(table) > 0 and re.match(self.underline,table[-1]):
5544	if line in ('',None):
5545	break;
5546	if line is None:
5547	raise EAsciiDoc,'closing [%s] underline expected' % self.defname
5548	table.append(reader.read())
5549	# EXPERIMENTAL: The number of lines in the table, requested by Benjamin Klum.
5550	self.attributes['rows'] = str(len(table))
5551	if self.check_msg: # Skip if table definition was marked invalid.
5552	message.warning('skipping [%s] table: %s' % (self.defname,self.check_msg))
5553	return
5554	self.push_blockname('table')
5555	# Generate colwidths and colspecs.
5556	self.build_colspecs()
5557	# Generate headrows, footrows, bodyrows.
5558	# Headrow, footrow and bodyrow data replaces same named attributes in
5559	# the table markup template. In order to ensure this data does not get
5560	# a second attribute substitution (which would interfere with any
5561	# already substituted inline passthroughs) unique placeholders are used
5562	# (the tab character does not appear elsewhere since it is expanded on
5563	# input) which are replaced after template attribute substitution.
5564	headrows = footrows = []
5565	bodyrows,table = self.split_rows(table)
5566	if table:
5567	headrows = bodyrows
5568	bodyrows,table = self.split_rows(table)
5569	if table:
5570	footrows,table = self.split_rows(table)
5571	if headrows:
5572	headrows = self.parse_rows(headrows, self.headrow, self.headdata)
5573	headrows = writer.newline.join(headrows)
5574	self.attributes['headrows'] = '\x07headrows\x07'
5575	if footrows:
5576	footrows = self.parse_rows(footrows, self.footrow, self.footdata)
5577	footrows = writer.newline.join(footrows)
5578	self.attributes['footrows'] = '\x07footrows\x07'
5579	bodyrows = self.parse_rows(bodyrows, self.bodyrow, self.bodydata)
5580	bodyrows = writer.newline.join(bodyrows)
5581	self.attributes['bodyrows'] = '\x07bodyrows\x07'
5582	table = subs_attrs(config.sections[self.template],self.attributes)
5583	table = writer.newline.join(table)
5584	# Before we finish replace the table head, foot and body place holders
5585	# with the real data.
5586	if headrows:
5587	table = table.replace('\x07headrows\x07', headrows, 1)
5588	if footrows:
5589	table = table.replace('\x07footrows\x07', footrows, 1)
5590	table = table.replace('\x07bodyrows\x07', bodyrows, 1)
5591	writer.write(table,trace='table')
5592	self.pop_blockname()
5593
5594	class Tables_OLD(AbstractBlocks):
5595	"""List of tables."""
5596	BLOCK_TYPE = Table_OLD
5597	PREFIX = 'old_tabledef-'
5598	def __init__(self):
5599	AbstractBlocks.__init__(self)
5600	def load(self,sections):
5601	AbstractBlocks.load(self,sections)
5602	def validate(self):
5603	# Does not call AbstractBlocks.validate().
5604	# Check we have a default table definition,
5605	for i in range(len(self.blocks)):
5606	if self.blocks[i].defname == 'old_tabledef-default':
5607	default = self.blocks[i]
5608	break
5609	else:
5610	raise EAsciiDoc,'missing section: [OLD_tabledef-default]'
5611	# Set default table defaults.
5612	if default.format is None: default.subs = 'fixed'
5613	# Propagate defaults to unspecified table parameters.
5614	for b in self.blocks:
5615	if b is not default:
5616	if b.fillchar is None: b.fillchar = default.fillchar
5617	if b.format is None: b.format = default.format
5618	if b.template is None: b.template = default.template
5619	if b.colspec is None: b.colspec = default.colspec
5620	if b.headrow is None: b.headrow = default.headrow
5621	if b.footrow is None: b.footrow = default.footrow
5622	if b.bodyrow is None: b.bodyrow = default.bodyrow
5623	if b.headdata is None: b.headdata = default.headdata
5624	if b.footdata is None: b.footdata = default.footdata
5625	if b.bodydata is None: b.bodydata = default.bodydata
5626	# Check all tables have valid fill character.
5627	for b in self.blocks:
5628	if not b.fillchar or len(b.fillchar) != 1:
5629	raise EAsciiDoc,'[%s] missing or illegal fillchar' % b.defname
5630	# Build combined tables delimiter patterns and assign defaults.
5631	delimiters = []
5632	for b in self.blocks:
5633	# Ruler is:
5634	# (ColStop,(ColWidth,FillChar+)?)+, FillChar+, TableWidth?
5635	b.delimiter = r'^(' + Table_OLD.COL_STOP \
5636	+ r'(\d\|' + re.escape(b.fillchar) + r')' \
5637	+ r')+' \
5638	+ re.escape(b.fillchar) + r'+' \
5639	+ '([\d\.]*)$'
5640	delimiters.append(b.delimiter)
5641	if not b.headrow:
5642	b.headrow = b.bodyrow
5643	if not b.footrow:
5644	b.footrow = b.bodyrow
5645	if not b.headdata:
5646	b.headdata = b.bodydata
5647	if not b.footdata:
5648	b.footdata = b.bodydata
5649	self.delimiters = re_join(delimiters)
5650	# Check table definitions are valid.
5651	for b in self.blocks:
5652	b.validate()
5653	if config.verbose:
5654	if b.check_msg:
5655	message.warning('[%s] table definition: %s' % (b.defname,b.check_msg))
5656
5657	# End of deprecated old table classes.
5658	#---------------------------------------------------------------------------
5659
5660	#---------------------------------------------------------------------------
5661	# filter and theme plugin commands.
5662	#---------------------------------------------------------------------------
5663	import shutil, zipfile
5664
5665	def die(msg):
5666	message.stderr(msg)
5667	sys.exit(1)
5668
5669	def extract_zip(zip_file, destdir):
5670	"""
5671	Unzip Zip file to destination directory.
5672	Throws exception if error occurs.
5673	"""
5674	zipo = zipfile.ZipFile(zip_file, 'r')
5675	try:
5676	for zi in zipo.infolist():
5677	outfile = zi.filename
5678	if not outfile.endswith('/'):
5679	d, outfile = os.path.split(outfile)
5680	directory = os.path.normpath(os.path.join(destdir, d))
5681	if not os.path.isdir(directory):
5682	os.makedirs(directory)
5683	outfile = os.path.join(directory, outfile)
5684	perms = (zi.external_attr >> 16) & 0777
5685	message.verbose('extracting: %s' % outfile)
5686	flags = os.O_CREAT \| os.O_WRONLY
5687	if sys.platform == 'win32':
5688	flags \|= os.O_BINARY
5689	if perms == 0:
5690	# Zip files created under Windows do not include permissions.
5691	fh = os.open(outfile, flags)
5692	else:
5693	fh = os.open(outfile, flags, perms)
5694	try:
5695	os.write(fh, zipo.read(zi.filename))
5696	finally:
5697	os.close(fh)
5698	finally:
5699	zipo.close()
5700
5701	def create_zip(zip_file, src, skip_hidden=False):
5702	"""
5703	Create Zip file. If src is a directory archive all contained files and
5704	subdirectories, if src is a file archive the src file.
5705	Files and directories names starting with . are skipped
5706	if skip_hidden is True.
5707	Throws exception if error occurs.
5708	"""
5709	zipo = zipfile.ZipFile(zip_file, 'w')
5710	try:
5711	if os.path.isfile(src):
5712	arcname = os.path.basename(src)
5713	message.verbose('archiving: %s' % arcname)
5714	zipo.write(src, arcname, zipfile.ZIP_DEFLATED)
5715	elif os.path.isdir(src):
5716	srcdir = os.path.abspath(src)
5717	if srcdir[-1] != os.path.sep:
5718	srcdir += os.path.sep
5719	for root, dirs, files in os.walk(srcdir):
5720	arcroot = os.path.abspath(root)[len(srcdir):]
5721	if skip_hidden:
5722	for d in dirs[:]:
5723	if d.startswith('.'):
5724	message.verbose('skipping: %s' % os.path.join(arcroot, d))
5725	del dirs[dirs.index(d)]
5726	for f in files:
5727	filename = os.path.join(root,f)
5728	arcname = os.path.join(arcroot, f)
5729	if skip_hidden and f.startswith('.'):
5730	message.verbose('skipping: %s' % arcname)
5731	continue
5732	message.verbose('archiving: %s' % arcname)
5733	zipo.write(filename, arcname, zipfile.ZIP_DEFLATED)
5734	else:
5735	raise ValueError,'src must specify directory or file: %s' % src
5736	finally:
5737	zipo.close()
5738
5739	class Plugin:
5740	"""
5741	--filter and --theme option commands.
5742	"""
5743	CMDS = ('install','remove','list','build')
5744
5745	type = None # 'backend', 'filter' or 'theme'.
5746
5747	@staticmethod
5748	def get_dir():
5749	"""
5750	Return plugins path (.asciidoc/filters or .asciidoc/themes) in user's
5751	home direcory or None if user home not defined.
5752	"""
5753	result = userdir()
5754	if result:
5755	result = os.path.join(result, '.asciidoc', Plugin.type+'s')
5756	return result
5757
5758	@staticmethod
5759	def install(args):
5760	"""
5761	Install plugin Zip file.
5762	args[0] is plugin zip file path.
5763	args[1] is optional destination plugins directory.
5764	"""
5765	if len(args) not in (1,2):
5766	die('invalid number of arguments: --%s install %s'
5767	% (Plugin.type, ' '.join(args)))
5768	zip_file = args[0]
5769	if not os.path.isfile(zip_file):
5770	die('file not found: %s' % zip_file)
5771	reo = re.match(r'^\w+',os.path.split(zip_file)[1])
5772	if not reo:
5773	die('file name does not start with legal %s name: %s'
5774	% (Plugin.type, zip_file))
5775	plugin_name = reo.group()
5776	if len(args) == 2:
5777	plugins_dir = args[1]
5778	if not os.path.isdir(plugins_dir):
5779	die('directory not found: %s' % plugins_dir)
5780	else:
5781	plugins_dir = Plugin.get_dir()
5782	if not plugins_dir:
5783	die('user home directory is not defined')
5784	plugin_dir = os.path.join(plugins_dir, plugin_name)
5785	if os.path.exists(plugin_dir):
5786	die('%s is already installed: %s' % (Plugin.type, plugin_dir))
5787	try:
5788	os.makedirs(plugin_dir)
5789	except Exception,e:
5790	die('failed to create %s directory: %s' % (Plugin.type, str(e)))
5791	try:
5792	extract_zip(zip_file, plugin_dir)
5793	except Exception,e:
5794	if os.path.isdir(plugin_dir):
5795	shutil.rmtree(plugin_dir)
5796	die('failed to extract %s: %s' % (Plugin.type, str(e)))
5797
5798	@staticmethod
5799	def remove(args):
5800	"""
5801	Delete plugin directory.
5802	args[0] is plugin name.
5803	args[1] is optional plugin directory (defaults to ~/.asciidoc/<plugin_name>).
5804	"""
5805	if len(args) not in (1,2):
5806	die('invalid number of arguments: --%s remove %s'
5807	% (Plugin.type, ' '.join(args)))
5808	plugin_name = args[0]
5809	if not re.match(r'^\w+$',plugin_name):
5810	die('illegal %s name: %s' % (Plugin.type, plugin_name))
5811	if len(args) == 2:
5812	d = args[1]
5813	if not os.path.isdir(d):
5814	die('directory not found: %s' % d)
5815	else:
5816	d = Plugin.get_dir()
5817	if not d:
5818	die('user directory is not defined')
5819	plugin_dir = os.path.join(d, plugin_name)
5820	if not os.path.isdir(plugin_dir):
5821	die('cannot find %s: %s' % (Plugin.type, plugin_dir))
5822	try:
5823	message.verbose('removing: %s' % plugin_dir)
5824	shutil.rmtree(plugin_dir)
5825	except Exception,e:
5826	die('failed to delete %s: %s' % (Plugin.type, str(e)))
5827
5828	@staticmethod
5829	def list(args):
5830	"""
5831	List all plugin directories (global and local).
5832	"""
5833	for d in [os.path.join(d, Plugin.type+'s') for d in config.get_load_dirs()]:
5834	if os.path.isdir(d):
5835	for f in os.walk(d).next()[1]:
5836	message.stdout(os.path.join(d,f))
5837
5838	@staticmethod
5839	def build(args):
5840	"""
5841	Create plugin Zip file.
5842	args[0] is Zip file name.
5843	args[1] is plugin directory.
5844	"""
5845	if len(args) != 2:
5846	die('invalid number of arguments: --%s build %s'
5847	% (Plugin.type, ' '.join(args)))
5848	zip_file = args[0]
5849	plugin_source = args[1]
5850	if not (os.path.isdir(plugin_source) or os.path.isfile(plugin_source)):
5851	die('plugin source not found: %s' % plugin_source)
5852	try:
5853	create_zip(zip_file, plugin_source, skip_hidden=True)
5854	except Exception,e:
5855	die('failed to create %s: %s' % (zip_file, str(e)))
5856
5857
5858	#---------------------------------------------------------------------------
5859	# Application code.
5860	#---------------------------------------------------------------------------
5861	# Constants
5862	# ---------
5863	APP_FILE = None # This file's full path.
5864	APP_DIR = None # This file's directory.
5865	USER_DIR = None # ~/.asciidoc
5866	# Global configuration files directory (set by Makefile build target).
5867	CONF_DIR = '/etc/asciidoc'
5868	HELP_FILE = 'help.conf' # Default (English) help file.
5869
5870	# Globals
5871	# -------
5872	document = Document() # The document being processed.
5873	config = Config() # Configuration file reader.
5874	reader = Reader() # Input stream line reader.
5875	writer = Writer() # Output stream line writer.
5876	message = Message() # Message functions.
5877	paragraphs = Paragraphs() # Paragraph definitions.
5878	lists = Lists() # List definitions.
5879	blocks = DelimitedBlocks() # DelimitedBlock definitions.
5880	tables_OLD = Tables_OLD() # Table_OLD definitions.
5881	tables = Tables() # Table definitions.
5882	macros = Macros() # Macro definitions.
5883	calloutmap = CalloutMap() # Coordinates callouts and callout list.
5884	trace = Trace() # Implements trace attribute processing.
5885
5886	### Used by asciidocapi.py ###
5887	# List of message strings written to stderr.
5888	messages = message.messages
5889
5890
5891	def asciidoc(backend, doctype, confiles, infile, outfile, options):
5892	"""Convert AsciiDoc document to DocBook document of type doctype
5893	The AsciiDoc document is read from file object src the translated
5894	DocBook file written to file object dst."""
5895	def load_conffiles(include=[], exclude=[]):
5896	# Load conf files specified on the command-line and by the conf-files attribute.
5897	files = document.attributes.get('conf-files','')
5898	files = [f.strip() for f in files.split('\|') if f.strip()]
5899	files += confiles
5900	if files:
5901	for f in files:
5902	if os.path.isfile(f):
5903	config.load_file(f, include=include, exclude=exclude)
5904	else:
5905	raise EAsciiDoc,'missing configuration file: %s' % f
5906	try:
5907	document.attributes['python'] = sys.executable
5908	for f in config.filters:
5909	if not config.find_config_dir('filters', f):
5910	raise EAsciiDoc,'missing filter: %s' % f
5911	if doctype not in (None,'article','manpage','book'):
5912	raise EAsciiDoc,'illegal document type'
5913	# Set processing options.
5914	for o in options:
5915	if o == '-c': config.dumping = True
5916	if o == '-s': config.header_footer = False
5917	if o == '-v': config.verbose = True
5918	document.update_attributes()
5919	if '-e' not in options:
5920	# Load asciidoc.conf files in two passes: the first for attributes
5921	# the second for everything. This is so that locally set attributes
5922	# available are in the global asciidoc.conf
5923	if not config.load_from_dirs('asciidoc.conf',include=['attributes']):
5924	raise EAsciiDoc,'configuration file asciidoc.conf missing'
5925	load_conffiles(include=['attributes'])
5926	config.load_from_dirs('asciidoc.conf')
5927	if infile != '<stdin>':
5928	indir = os.path.dirname(infile)
5929	config.load_file('asciidoc.conf', indir,
5930	include=['attributes','titles','specialchars'])
5931	else:
5932	load_conffiles(include=['attributes','titles','specialchars'])
5933	document.update_attributes()
5934	# Check the infile exists.
5935	if infile != '<stdin>':
5936	if not os.path.isfile(infile):
5937	raise EAsciiDoc,'input file %s missing' % infile
5938	document.infile = infile
5939	AttributeList.initialize()
5940	# Open input file and parse document header.
5941	reader.tabsize = config.tabsize
5942	reader.open(infile)
5943	has_header = document.parse_header(doctype,backend)
5944	# doctype is now finalized.
5945	document.attributes['doctype-'+document.doctype] = ''
5946	config.set_theme_attributes()
5947	# Load backend configuration files.
5948	if '-e' not in options:
5949	f = document.backend + '.conf'
5950	conffile = config.load_backend()
5951	if not conffile:
5952	raise EAsciiDoc,'missing backend conf file: %s' % f
5953	document.attributes['backend-confdir'] = os.path.dirname(conffile)
5954	# backend is now known.
5955	document.attributes['backend-'+document.backend] = ''
5956	document.attributes[document.backend+'-'+document.doctype] = ''
5957	doc_conffiles = []
5958	if '-e' not in options:
5959	# Load filters and language file.
5960	config.load_filters()
5961	document.load_lang()
5962	if infile != '<stdin>':
5963	# Load local conf files (files in the source file directory).
5964	config.load_file('asciidoc.conf', indir)
5965	config.load_backend([indir])
5966	config.load_filters([indir])
5967	# Load document specific configuration files.
5968	f = os.path.splitext(infile)[0]
5969	doc_conffiles = [
5970	f for f in (f+'.conf', f+'-'+document.backend+'.conf')
5971	if os.path.isfile(f) ]
5972	for f in doc_conffiles:
5973	config.load_file(f)
5974	load_conffiles()
5975	# Build asciidoc-args attribute.
5976	args = ''
5977	# Add custom conf file arguments.
5978	for f in doc_conffiles + confiles:
5979	args += ' --conf-file "%s"' % f
5980	# Add command-line and header attributes.
5981	attrs = {}
5982	attrs.update(AttributeEntry.attributes)
5983	attrs.update(config.cmd_attrs)
5984	if 'title' in attrs: # Don't pass the header title.
5985	del attrs['title']
5986	for k,v in attrs.items():
5987	if v:
5988	args += ' --attribute "%s=%s"' % (k,v)
5989	else:
5990	args += ' --attribute "%s"' % k
5991	document.attributes['asciidoc-args'] = args
5992	# Build outfile name.
5993	if outfile is None:
5994	outfile = os.path.splitext(infile)[0] + '.' + document.backend
5995	if config.outfilesuffix:
5996	# Change file extension.
5997	outfile = os.path.splitext(outfile)[0] + config.outfilesuffix
5998	document.outfile = outfile
5999	# Document header attributes override conf file attributes.
6000	document.attributes.update(AttributeEntry.attributes)
6001	document.update_attributes()
6002	# Configuration is fully loaded.
6003	config.expand_all_templates()
6004	# Check configuration for consistency.
6005	config.validate()
6006	# Initialize top level block name.
6007	if document.attributes.get('blockname'):
6008	AbstractBlock.blocknames.append(document.attributes['blockname'])
6009	paragraphs.initialize()
6010	lists.initialize()
6011	if config.dumping:
6012	config.dump()
6013	else:
6014	writer.newline = config.newline
6015	try:
6016	writer.open(outfile, reader.bom)
6017	try:
6018	document.translate(has_header) # Generate the output.
6019	finally:
6020	writer.close()
6021	finally:
6022	reader.closefile()
6023	except KeyboardInterrupt:
6024	raise
6025	except Exception,e:
6026	# Cleanup.
6027	if outfile and outfile != '<stdout>' and os.path.isfile(outfile):
6028	os.unlink(outfile)
6029	# Build and print error description.
6030	msg = 'FAILED: '
6031	if reader.cursor:
6032	msg = message.format('', msg)
6033	if isinstance(e, EAsciiDoc):
6034	message.stderr('%s%s' % (msg,str(e)))
6035	else:
6036	if __name__ == '__main__':
6037	message.stderr(msg+'unexpected error:')
6038	message.stderr('-'*60)
6039	traceback.print_exc(file=sys.stderr)
6040	message.stderr('-'*60)
6041	else:
6042	message.stderr('%sunexpected error: %s' % (msg,str(e)))
6043	sys.exit(1)
6044
6045	def usage(msg=''):
6046	if msg:
6047	message.stderr(msg)
6048	show_help('default', sys.stderr)
6049
6050	def show_help(topic, f=None):
6051	"""Print help topic to file object f."""
6052	if f is None:
6053	f = sys.stdout
6054	# Select help file.
6055	lang = config.cmd_attrs.get('lang')
6056	if lang and lang != 'en':
6057	help_file = 'help-' + lang + '.conf'
6058	else:
6059	help_file = HELP_FILE
6060	# Print [topic] section from help file.
6061	config.load_from_dirs(help_file)
6062	if len(config.sections) == 0:
6063	# Default to English if specified language help files not found.
6064	help_file = HELP_FILE
6065	config.load_from_dirs(help_file)
6066	if len(config.sections) == 0:
6067	message.stderr('no help topics found')
6068	sys.exit(1)
6069	n = 0
6070	for k in config.sections:
6071	if re.match(re.escape(topic), k):
6072	n += 1
6073	lines = config.sections[k]
6074	if n == 0:
6075	if topic != 'topics':
6076	message.stderr('help topic not found: [%s] in %s' % (topic, help_file))
6077	message.stderr('available help topics: %s' % ', '.join(config.sections.keys()))
6078	sys.exit(1)
6079	elif n > 1:
6080	message.stderr('ambiguous help topic: %s' % topic)
6081	else:
6082	for line in lines:
6083	print >>f, line
6084
6085	### Used by asciidocapi.py ###
6086	def execute(cmd,opts,args):
6087	"""
6088	Execute asciidoc with command-line options and arguments.
6089	cmd is asciidoc command or asciidoc.py path.
6090	opts and args conform to values returned by getopt.getopt().
6091	Raises SystemExit if an error occurs.
6092
6093	Doctests:
6094
6095	1. Check execution:
6096
6097	>>> import StringIO
6098	>>> infile = StringIO.StringIO('Hello {author}')
6099	>>> outfile = StringIO.StringIO()
6100	>>> opts = []
6101	>>> opts.append(('--backend','html4'))
6102	>>> opts.append(('--no-header-footer',None))
6103	>>> opts.append(('--attribute','author=Joe Bloggs'))
6104	>>> opts.append(('--out-file',outfile))
6105	>>> execute(__file__, opts, [infile])
6106	>>> print outfile.getvalue()
6107	<p>Hello <strong>Joe Bloggs</strong></p>
6108
6109	>>>
6110
6111	"""
6112	config.init(cmd)
6113	if len(args) > 1:
6114	usage('Too many arguments')
6115	sys.exit(1)
6116	backend = None
6117	doctype = None
6118	confiles = []
6119	outfile = None
6120	options = []
6121	help_option = False
6122	for o,v in opts:
6123	if o in ('--help','-h'):
6124	help_option = True
6125	#DEPRECATED: --unsafe option.
6126	if o == '--unsafe':
6127	document.safe = False
6128	if o == '--safe':
6129	document.safe = True
6130	if o == '--version':
6131	print('asciidoc %s' % VERSION)
6132	sys.exit(0)
6133	if o in ('-b','--backend'):
6134	backend = v
6135	if o in ('-c','--dump-conf'):
6136	options.append('-c')
6137	if o in ('-d','--doctype'):
6138	doctype = v
6139	if o in ('-e','--no-conf'):
6140	options.append('-e')
6141	if o in ('-f','--conf-file'):
6142	confiles.append(v)
6143	if o == '--filter':
6144	config.filters.append(v)
6145	if o in ('-n','--section-numbers'):
6146	o = '-a'
6147	v = 'numbered'
6148	if o == '--theme':
6149	o = '-a'
6150	v = 'theme='+v
6151	if o in ('-a','--attribute'):
6152	e = parse_entry(v, allow_name_only=True)
6153	if not e:
6154	usage('Illegal -a option: %s' % v)
6155	sys.exit(1)
6156	k,v = e
6157	# A @ suffix denotes don't override existing document attributes.
6158	if v and v[-1] == '@':
6159	document.attributes[k] = v[:-1]
6160	else:
6161	config.cmd_attrs[k] = v
6162	if o in ('-o','--out-file'):
6163	outfile = v
6164	if o in ('-s','--no-header-footer'):
6165	options.append('-s')
6166	if o in ('-v','--verbose'):
6167	options.append('-v')
6168	if help_option:
6169	if len(args) == 0:
6170	show_help('default')
6171	else:
6172	show_help(args[-1])
6173	sys.exit(0)
6174	if len(args) == 0 and len(opts) == 0:
6175	usage()
6176	sys.exit(0)
6177	if len(args) == 0:
6178	usage('No source file specified')
6179	sys.exit(1)
6180	stdin,stdout = sys.stdin,sys.stdout
6181	try:
6182	infile = args[0]
6183	if infile == '-':
6184	infile = '<stdin>'
6185	elif isinstance(infile, str):
6186	infile = os.path.abspath(infile)
6187	else: # Input file is file object from API call.
6188	sys.stdin = infile
6189	infile = '<stdin>'
6190	if outfile == '-':
6191	outfile = '<stdout>'
6192	elif isinstance(outfile, str):
6193	outfile = os.path.abspath(outfile)
6194	elif outfile is None:
6195	if infile == '<stdin>':
6196	outfile = '<stdout>'
6197	else: # Output file is file object from API call.
6198	sys.stdout = outfile
6199	outfile = '<stdout>'
6200	# Do the work.
6201	asciidoc(backend, doctype, confiles, infile, outfile, options)
6202	if document.has_errors:
6203	sys.exit(1)
6204	finally:
6205	sys.stdin,sys.stdout = stdin,stdout
6206
6207	if __name__ == '__main__':
6208	# Process command line options.
6209	import getopt
6210	try:
6211	#DEPRECATED: --unsafe option.
6212	opts,args = getopt.getopt(sys.argv[1:],
6213	'a:b:cd:ef:hno:svw:',
6214	['attribute=','backend=','conf-file=','doctype=','dump-conf',
6215	'help','no-conf','no-header-footer','out-file=',
6216	'section-numbers','verbose','version','safe','unsafe',
6217	'doctest','filter=','theme='])
6218	except getopt.GetoptError:
6219	message.stderr('illegal command options')
6220	sys.exit(1)
6221	opt_names = [opt[0] for opt in opts]
6222	if '--doctest' in opt_names:
6223	# Run module doctests.
6224	import doctest
6225	options = doctest.NORMALIZE_WHITESPACE + doctest.ELLIPSIS
6226	failures,tries = doctest.testmod(optionflags=options)
6227	if failures == 0:
6228	message.stderr('All doctests passed')
6229	sys.exit(0)
6230	else:
6231	sys.exit(1)
6232	# Look for plugin management commands.
6233	count = 0
6234	for o,v in opts:
6235	if o in ('-b','--backend','--filter','--theme'):
6236	if o == '-b':
6237	o = '--backend'
6238	plugin = o[2:]
6239	cmd = v
6240	if cmd not in Plugin.CMDS:
6241	continue
6242	count += 1
6243	if count > 1:
6244	die('--backend, --filter and --theme options are mutually exclusive')
6245	if count == 1:
6246	# Execute plugin management commands.
6247	if not cmd:
6248	die('missing --%s command' % plugin)
6249	if cmd not in Plugin.CMDS:
6250	die('illegal --%s command: %s' % (plugin, cmd))
6251	Plugin.type = plugin
6252	config.init(sys.argv[0])
6253	config.verbose = bool(set(['-v','--verbose']) & set(opt_names))
6254	getattr(Plugin,cmd)(args)
6255	else:
6256	# Execute asciidoc.
6257	try:
6258	execute(sys.argv[0],opts,args)
6259	except KeyboardInterrupt:
6260	sys.exit(1)

Note: See TracBrowser for help on using the repository browser.

Download in other formats: