Clean up & improve makerst.py

Man this file even had some semicolons in it.

I cleaned up the entire file, while it's still pretty ugly it's much better now.
I also added type checks so it passes mypy --strict.
make_type now throws a warning on unresolved type references, which there are a bunch of. I'm not responsible for fixing those though.
Also some more hardening against crashes. For example XML tags without content won't cause crashes now.
Functionality has not been modified as far as I can tell.

Update Makefile for Python 3

Fix ordering issues related to enums & constants
This commit is contained in:
Pieter-Jan Briers 2018-12-26 15:57:51 +01:00
parent 10e9221c49
commit 15a3d16d08
2 changed files with 174 additions and 176 deletions

View File

@ -24,5 +24,5 @@ rst:
rm -rf $(OUTPUTDIR)/rst rm -rf $(OUTPUTDIR)/rst
mkdir -p $(OUTPUTDIR)/rst mkdir -p $(OUTPUTDIR)/rst
pushd $(OUTPUTDIR)/rst pushd $(OUTPUTDIR)/rst
python $(TOOLSDIR)/makerst.py $(CLASSES) python3 $(TOOLSDIR)/makerst.py $(CLASSES)
popd popd

View File

@ -1,73 +1,107 @@
#!/usr/bin/env python #!/usr/bin/env python3
# -*- coding: utf-8 -*-
import codecs
import sys import sys
import os import os
import re import re
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from collections import defaultdict
input_list = [] # Uncomment to do type checks. I have it commented out so it works below Python 3.5
cur_file = "" #from typing import List, Dict, TextIO, Tuple, Iterable, Optional, DefaultDict
input_list = [] # type: List[str]
current_reading_class = ""
class_names = [] # type: List[str]
classes = {} # type: Dict[str, ET.Element]
# http(s)://docs.godotengine.org/<langcode>/<tag>/path/to/page.html(#fragment-tag) # http(s)://docs.godotengine.org/<langcode>/<tag>/path/to/page.html(#fragment-tag)
godot_docs_pattern = re.compile('^http(?:s)?:\/\/docs\.godotengine\.org\/(?:[a-zA-Z0-9\.\-_]*)\/(?:[a-zA-Z0-9\.\-_]*)\/(.*)\.html(#.*)?$') GODOT_DOCS_PATTERN = re.compile(r'^http(?:s)?://docs\.godotengine\.org/(?:[a-zA-Z0-9.\-_]*)/(?:[a-zA-Z0-9.\-_]*)/(.*)\.html(#.*)?$')
for arg in sys.argv[1:]:
if arg.endswith(os.sep):
arg = arg[:-1]
input_list.append(arg)
if len(input_list) < 1:
print('usage: makerst.py <path to folders> and/or <path to .xml files> (order of arguments irrelevant)')
print('example: makerst.py "../../modules/" "../classes" path_to/some_class.xml')
sys.exit(0)
def validate_tag(elem, tag): def main(): # type: () -> None
if elem.tag != tag: global current_reading_class
print("Tag mismatch, expected '" + tag + "', got " + elem.tag) for arg in sys.argv[1:]:
sys.exit(255) if arg.endswith(os.sep):
arg = arg[:-1]
input_list.append(arg)
if len(input_list) < 1:
print('usage: makerst.py <path to folders> and/or <path to .xml files> (order of arguments irrelevant)')
print('example: makerst.py "../../modules/" "../classes" path_to/some_class.xml')
sys.exit(1)
file_list = [] # type: List[str]
for path in input_list:
if os.path.basename(path) == 'modules':
for subdir, dirs, _ in os.walk(path):
if 'doc_classes' in dirs:
doc_dir = os.path.join(subdir, 'doc_classes')
class_file_names = [f for f in os.listdir(doc_dir) if f.endswith('.xml')]
file_list += [os.path.join(doc_dir, f) for f in class_file_names]
elif os.path.isdir(path):
file_list += [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.xml')]
elif os.path.isfile(path):
if not path.endswith(".xml"):
print("Got non-.xml file '{}' in input, skipping.".format(path))
continue
file_list.append(path)
class_names = [] for cur_file in file_list:
classes = {} try:
tree = ET.parse(cur_file)
except ET.ParseError as e:
print("Parse error reading file '{}': {}".format(cur_file, e))
sys.exit(1)
doc = tree.getroot()
if 'version' not in doc.attrib:
print("Version missing from 'doc'")
sys.exit(255)
name = doc.attrib["name"]
if name in classes:
continue
class_names.append(name)
classes[name] = doc
class_names.sort()
# Don't make class list for Sphinx, :toctree: handles it
# make_class_list(class_names, 2)
for c in class_names:
current_reading_class = c
make_rst_class(classes[c])
def ul_string(str, ul): def make_class_list(class_list, columns): # type: (List[str], int) -> None
str += "\n" # This function is no longer used.
for i in range(len(str) - 1): f = open('class_list.rst', 'w', encoding='utf-8')
str += ul col_max = len(class_list) // columns + 1
str += "\n"
return str
def make_class_list(class_list, columns):
f = codecs.open('class_list.rst', 'wb', 'utf-8')
prev = 0
col_max = len(class_list) / columns + 1
print(('col max is ', col_max)) print(('col max is ', col_max))
col_count = 0 fit_columns = [] # type: List[List[str]]
row_count = 0
last_initial = ''
fit_columns = []
for n in range(0, columns): for _ in range(0, columns):
fit_columns += [[]] fit_columns.append([])
indexers = [] indexers = [] # type List[str]
last_initial = '' last_initial = ''
idx = 0 for (idx, name) in enumerate(class_list):
for n in class_list: col = idx // col_max
col = idx / col_max
if col >= columns: if col >= columns:
col = columns - 1 col = columns - 1
fit_columns[col] += [n] fit_columns[col].append(name)
idx += 1 idx += 1
if n[:1] != last_initial: if name[:1] != last_initial:
indexers += [n] indexers.append(name)
last_initial = n[:1] last_initial = name[:1]
row_max = 0 row_max = 0
f.write("\n") f.write("\n")
@ -111,7 +145,7 @@ def make_class_list(class_list, columns):
f.close() f.close()
def rstize_text(text, cclass): def rstize_text(text, cclass): # type: (str, str) -> str
# Linebreak + tabs in the XML should become two line breaks unless in a "codeblock" # Linebreak + tabs in the XML should become two line breaks unless in a "codeblock"
pos = 0 pos = 0
while True: while True:
@ -209,7 +243,7 @@ def rstize_text(text, cclass):
escape_post = False escape_post = False
if tag_text in class_names: if tag_text in classes:
tag_text = make_type(tag_text) tag_text = make_type(tag_text)
escape_post = True escape_post = True
else: # command else: # command
@ -228,17 +262,15 @@ def rstize_text(text, cclass):
elif inside_code: elif inside_code:
tag_text = '[' + tag_text + ']' tag_text = '[' + tag_text + ']'
elif cmd.find('html') == 0: elif cmd.find('html') == 0:
cmd = tag_text[:space_pos]
param = tag_text[space_pos + 1:] param = tag_text[space_pos + 1:]
tag_text = param tag_text = param
elif cmd.find('method') == 0 or cmd.find('member') == 0 or cmd.find('signal') == 0: elif cmd.find('method') == 0 or cmd.find('member') == 0 or cmd.find('signal') == 0:
cmd = tag_text[:space_pos]
param = tag_text[space_pos + 1:] param = tag_text[space_pos + 1:]
if param.find('.') != -1: if param.find('.') != -1:
ss = param.split('.') ss = param.split('.')
if len(ss) > 2: if len(ss) > 2:
sys.exit("Bad reference: '" + param + "' in file: " + cur_file) sys.exit("Bad reference: '" + param + "' in class: " + current_reading_class)
(class_param, method_param) = ss (class_param, method_param) = ss
tag_text = ':ref:`' + class_param + '.' + method_param + '<class_' + class_param + '_' + method_param + '>`' tag_text = ':ref:`' + class_param + '.' + method_param + '<class_' + class_param + '_' + method_param + '>`'
else: else:
@ -309,15 +341,15 @@ def rstize_text(text, cclass):
return text return text
def format_table(f, pp): def format_table(f, pp): # type: (TextIO, Iterable[Tuple[str, ...]]) -> None
longest_t = 0 longest_t = 0
longest_s = 0 longest_s = 0
for s in pp: for s in pp:
sl = len(s[0]) sl = len(s[0])
if (sl > longest_s): if sl > longest_s:
longest_s = sl longest_s = sl
tl = len(s[1]) tl = len(s[1])
if (tl > longest_t): if tl > longest_t:
longest_t = tl longest_t = tl
sep = "+" sep = "+"
@ -330,25 +362,24 @@ def format_table(f, pp):
f.write(sep) f.write(sep)
for s in pp: for s in pp:
rt = s[0] rt = s[0]
while (len(rt) < longest_s): while len(rt) < longest_s:
rt += " " rt += " "
st = s[1] st = s[1]
while (len(st) < longest_t): while len(st) < longest_t:
st += " " st += " "
f.write("| " + rt + " | " + st + " |\n") f.write("| " + rt + " | " + st + " |\n")
f.write(sep) f.write(sep)
f.write('\n') f.write('\n')
def make_type(t): def make_type(t): # type: (str) -> str
global class_names if t in classes:
if t in class_names:
return ':ref:`' + t + '<class_' + t + '>`' return ':ref:`' + t + '<class_' + t + '>`'
print("Warning: unresolved type reference '{}' in class '{}'".format(t, current_reading_class))
return t return t
def make_enum(t): def make_enum(t): # type: (str) -> str
global class_names
p = t.find(".") p = t.find(".")
# Global enums such as Error are relative to @GlobalScope. # Global enums such as Error are relative to @GlobalScope.
if p >= 0: if p >= 0:
@ -368,39 +399,41 @@ def make_enum(t):
def make_method( def make_method(
f, f, # type: TextIO
cname, cname, # type: str
method_data, method_data, # type: ET.Element
declare, declare, # type: bool
event=False, event=False, # type: bool
pp=None pp=None # type: Optional[List[Tuple[str, str]]]
): ): # type: (...) -> None
if (declare or pp is None): if declare or pp is None:
t = '- ' t = '- '
else: else:
t = "" t = ""
ret_type = 'void' argidx = [] # type: List[int]
args = list(method_data) args = list(method_data)
mdata = {} mdata = {} # type: Dict[int, ET.Element]
mdata['argidx'] = [] for arg in args:
for a in args: if arg.tag == 'return':
if a.tag == 'return':
idx = -1 idx = -1
elif a.tag == 'argument': elif arg.tag == 'argument':
idx = int(a.attrib['index']) idx = int(arg.attrib['index'])
else: else:
continue continue
mdata['argidx'].append(idx) argidx.append(idx)
mdata[idx] = a mdata[idx] = arg
if not event: if not event:
if -1 in mdata['argidx']: if -1 in argidx:
if 'enum' in mdata[-1].attrib: if 'enum' in mdata[-1].attrib:
t += make_enum(mdata[-1].attrib['enum']) t += make_enum(mdata[-1].attrib['enum'])
else: else:
t += make_type(mdata[-1].attrib['type']) if mdata[-1].attrib['type'] == 'void':
t += 'void'
else:
t += make_type(mdata[-1].attrib['type'])
else: else:
t += 'void' t += 'void'
t += ' ' t += ' '
@ -412,8 +445,7 @@ def make_method(
s = ':ref:`' + method_data.attrib['name'] + '<class_' + cname + "_" + method_data.attrib['name'] + '>` ' s = ':ref:`' + method_data.attrib['name'] + '<class_' + cname + "_" + method_data.attrib['name'] + '>` '
s += '**(**' s += '**(**'
argfound = False for a in argidx:
for a in mdata['argidx']:
arg = mdata[a] arg = mdata[a]
if a < 0: if a < 0:
continue continue
@ -439,8 +471,8 @@ def make_method(
if 'qualifiers' in method_data.attrib: if 'qualifiers' in method_data.attrib:
s += ' ' + method_data.attrib['qualifiers'] s += ' ' + method_data.attrib['qualifiers']
if (not declare): if not declare:
if (pp != None): if pp is not None:
pp.append((t, s)) pp.append((t, s))
else: else:
f.write("- " + t + " " + s + "\n") f.write("- " + t + " " + s + "\n")
@ -449,12 +481,12 @@ def make_method(
def make_properties( def make_properties(
f, f, # type: TextIO
cname, cname, # type: str
prop_data, prop_data, # type: ET.Element
description=False, description=False, # type: bool
pp=None pp=None # type: Optional[List[Tuple[str, str]]]
): ): # type: (...) -> None
t = "" t = ""
if 'enum' in prop_data.attrib: if 'enum' in prop_data.attrib:
t += make_enum(prop_data.attrib['enum']) t += make_enum(prop_data.attrib['enum'])
@ -471,7 +503,7 @@ def make_properties(
else: else:
s = ':ref:`' + prop_data.attrib['name'] + '<class_' + cname + "_" + prop_data.attrib['name'] + '>`' s = ':ref:`' + prop_data.attrib['name'] + '<class_' + cname + "_" + prop_data.attrib['name'] + '>`'
if (pp != None): if pp is not None:
pp.append((t, s)) pp.append((t, s))
elif description: elif description:
f.write('- ' + t + ' ' + s + '\n\n') f.write('- ' + t + ' ' + s + '\n\n')
@ -479,14 +511,14 @@ def make_properties(
format_table(f, setget) format_table(f, setget)
def make_heading(title, underline): def make_heading(title, underline): # type: (str, str) -> str
return title + '\n' + underline * len(title) + "\n\n" return title + '\n' + (underline * len(title)) + "\n\n"
def make_rst_class(node): def make_rst_class(node): # type: (ET.Element) -> None
name = node.attrib['name'] name = node.attrib['name']
f = codecs.open("class_" + name.lower() + '.rst', 'wb', 'utf-8') f = open("class_" + name.lower() + '.rst', 'w', encoding='utf-8')
# Warn contributors not to edit this file directly # Warn contributors not to edit this file directly
f.write(".. Generated automatically by doc/tools/makerst.py in Godot's source tree.\n") f.write(".. Generated automatically by doc/tools/makerst.py in Godot's source tree.\n")
@ -502,31 +534,31 @@ def make_rst_class(node):
inh = node.attrib['inherits'].strip() inh = node.attrib['inherits'].strip()
f.write('**Inherits:** ') f.write('**Inherits:** ')
first = True first = True
while (inh in classes): while inh in classes:
if (not first): if not first:
f.write(" **<** ") f.write(" **<** ")
else: else:
first = False first = False
f.write(make_type(inh)) f.write(make_type(inh))
inode = classes[inh] inode = classes[inh]
if ('inherits' in inode.attrib): if 'inherits' in inode.attrib:
inh = inode.attrib['inherits'].strip() inh = inode.attrib['inherits'].strip()
else: else:
inh = None break
f.write("\n\n") f.write("\n\n")
# Descendents # Descendents
inherited = [] inherited = []
for cn in classes: for cn in class_names:
c = classes[cn] c = classes[cn]
if 'inherits' in c.attrib: if 'inherits' in c.attrib:
if (c.attrib['inherits'].strip() == name): if c.attrib['inherits'].strip() == name:
inherited.append(c.attrib['name']) inherited.append(c.attrib['name'])
if (len(inherited)): if len(inherited):
f.write('**Inherited By:** ') f.write('**Inherited By:** ')
for i in range(len(inherited)): for i in range(len(inherited)):
if (i > 0): if i > 0:
f.write(", ") f.write(", ")
f.write(make_type(inherited[i])) f.write(make_type(inherited[i]))
f.write("\n\n") f.write("\n\n")
@ -538,46 +570,46 @@ def make_rst_class(node):
# Brief description # Brief description
f.write(make_heading('Brief Description', '-')) f.write(make_heading('Brief Description', '-'))
briefd = node.find('brief_description') briefd = node.find('brief_description')
if briefd != None: if briefd is not None and briefd.text is not None:
f.write(rstize_text(briefd.text.strip(), name) + "\n\n") f.write(rstize_text(briefd.text.strip(), name) + "\n\n")
# Properties overview # Properties overview
members = node.find('members') members = node.find('members')
if members != None and len(list(members)) > 0: if members is not None and len(list(members)) > 0:
f.write(make_heading('Properties', '-')) f.write(make_heading('Properties', '-'))
ml = [] ml = [] # type: List[Tuple[str, str]]
for m in list(members): for m in members:
make_properties(f, name, m, False, ml) make_properties(f, name, m, False, ml)
format_table(f, ml) format_table(f, ml)
# Methods overview # Methods overview
methods = node.find('methods') methods = node.find('methods')
if methods != None and len(list(methods)) > 0: if methods is not None and len(list(methods)) > 0:
f.write(make_heading('Methods', '-')) f.write(make_heading('Methods', '-'))
ml = [] ml = []
for m in list(methods): for m in methods:
make_method(f, name, m, False, False, ml) make_method(f, name, m, False, False, ml)
format_table(f, ml) format_table(f, ml)
# Theme properties # Theme properties
theme_items = node.find('theme_items') theme_items = node.find('theme_items')
if theme_items != None and len(list(theme_items)) > 0: if theme_items is not None and len(list(theme_items)) > 0:
f.write(make_heading('Theme Properties', '-')) f.write(make_heading('Theme Properties', '-'))
ml = [] ml = []
for m in list(theme_items): for m in theme_items:
make_properties(f, name, m, False, ml) make_properties(f, name, m, False, ml)
format_table(f, ml) format_table(f, ml)
# Signals # Signals
events = node.find('signals') events = node.find('signals')
if events != None and len(list(events)) > 0: if events is not None and len(list(events)) > 0:
f.write(make_heading('Signals', '-')) f.write(make_heading('Signals', '-'))
for m in list(events): for m in events:
f.write(".. _class_" + name + "_" + m.attrib['name'] + ":\n\n") f.write(".. _class_" + name + "_" + m.attrib['name'] + ":\n\n")
make_method(f, name, m, True, True) make_method(f, name, m, True, True)
f.write('\n') f.write('\n')
d = m.find('description') d = m.find('description')
if d is None or d.text.strip() == '': if d is None or d.text is None or d.text.strip() == '':
continue continue
f.write(rstize_text(d.text.strip(), name)) f.write(rstize_text(d.text.strip(), name))
f.write("\n\n") f.write("\n\n")
@ -585,13 +617,15 @@ def make_rst_class(node):
# Constants and enums # Constants and enums
constants = node.find('constants') constants = node.find('constants')
consts = [] consts = []
enum_names = set() enum_names = []
enums = [] enums = defaultdict(list) # type: DefaultDict[str, List[ET.Element]]
if constants != None and len(list(constants)) > 0: if constants is not None and len(list(constants)) > 0:
for c in list(constants): for c in constants:
if 'enum' in c.attrib: if 'enum' in c.attrib:
enum_names.add(c.attrib['enum']) ename = c.attrib['enum']
enums.append(c) if ename not in enums:
enum_names.append(ename)
enums[ename].append(c)
else: else:
consts.append(c) consts.append(c)
@ -601,43 +635,42 @@ def make_rst_class(node):
for e in enum_names: for e in enum_names:
f.write(".. _enum_" + name + "_" + e + ":\n\n") f.write(".. _enum_" + name + "_" + e + ":\n\n")
f.write("enum **" + e + "**:\n\n") f.write("enum **" + e + "**:\n\n")
for c in enums: for c in enums[e]:
if c.attrib['enum'] != e:
continue
s = '- ' s = '- '
s += '**' + c.attrib['name'] + '**' s += '**' + c.attrib['name'] + '**'
if 'value' in c.attrib: if 'value' in c.attrib:
s += ' = **' + c.attrib['value'] + '**' s += ' = **' + c.attrib['value'] + '**'
if c.text.strip() != '': if c.text is not None and c.text.strip() != '':
s += ' --- ' + rstize_text(c.text.strip(), name) s += ' --- ' + rstize_text(c.text.strip(), name)
f.write(s + '\n\n') f.write(s + '\n\n')
# Constants # Constants
if len(consts) > 0: if len(consts) > 0:
f.write(make_heading('Constants', '-')) f.write(make_heading('Constants', '-'))
for c in list(consts): for c in consts:
s = '- ' s = '- '
s += '**' + c.attrib['name'] + '**' s += '**' + c.attrib['name'] + '**'
if 'value' in c.attrib: if 'value' in c.attrib:
s += ' = **' + c.attrib['value'] + '**' s += ' = **' + c.attrib['value'] + '**'
if c.text.strip() != '': if c.text is not None and c.text.strip() != '':
s += ' --- ' + rstize_text(c.text.strip(), name) s += ' --- ' + rstize_text(c.text.strip(), name)
f.write(s + '\n\n') f.write(s + '\n\n')
# Class description # Class description
descr = node.find('description') descr = node.find('description')
if descr != None and descr.text.strip() != '': if descr is not None and descr.text is not None and descr.text.strip() != '':
f.write(make_heading('Description', '-')) f.write(make_heading('Description', '-'))
f.write(rstize_text(descr.text.strip(), name) + "\n\n") f.write(rstize_text(descr.text.strip(), name) + "\n\n")
# Online tutorials # Online tutorials
global godot_docs_pattern
tutorials = node.find('tutorials') tutorials = node.find('tutorials')
if tutorials != None and len(tutorials) > 0: if tutorials is not None and len(tutorials) > 0:
f.write(make_heading('Tutorials', '-')) f.write(make_heading('Tutorials', '-'))
for t in tutorials: for t in tutorials:
if t.text is None:
continue
link = t.text.strip() link = t.text.strip()
match = godot_docs_pattern.search(link); match = GODOT_DOCS_PATTERN.search(link)
if match: if match:
groups = match.groups() groups = match.groups()
if match.lastindex == 2: if match.lastindex == 2:
@ -658,63 +691,28 @@ def make_rst_class(node):
# Property descriptions # Property descriptions
members = node.find('members') members = node.find('members')
if members != None and len(list(members)) > 0: if members is not None and len(list(members)) > 0:
f.write(make_heading('Property Descriptions', '-')) f.write(make_heading('Property Descriptions', '-'))
for m in list(members): for m in members:
f.write(".. _class_" + name + "_" + m.attrib['name'] + ":\n\n") f.write(".. _class_" + name + "_" + m.attrib['name'] + ":\n\n")
make_properties(f, name, m, True) make_properties(f, name, m, True)
if m.text.strip() != '': if m.text is not None and m.text.strip() != '':
f.write(rstize_text(m.text.strip(), name)) f.write(rstize_text(m.text.strip(), name))
f.write('\n\n') f.write('\n\n')
# Method descriptions # Method descriptions
methods = node.find('methods') methods = node.find('methods')
if methods != None and len(list(methods)) > 0: if methods is not None and len(list(methods)) > 0:
f.write(make_heading('Method Descriptions', '-')) f.write(make_heading('Method Descriptions', '-'))
for m in list(methods): for m in methods:
f.write(".. _class_" + name + "_" + m.attrib['name'] + ":\n\n") f.write(".. _class_" + name + "_" + m.attrib['name'] + ":\n\n")
make_method(f, name, m, True) make_method(f, name, m, True)
f.write('\n') f.write('\n')
d = m.find('description') d = m.find('description')
if d is None or d.text.strip() == '': if d is None or d.text is None or d.text.strip() == '':
continue continue
f.write(rstize_text(d.text.strip(), name)) f.write(rstize_text(d.text.strip(), name))
f.write("\n\n") f.write("\n\n")
if __name__ == '__main__':
file_list = [] main()
for path in input_list:
if os.path.basename(path) == 'modules':
for subdir, dirs, _ in os.walk(path):
if 'doc_classes' in dirs:
doc_dir = os.path.join(subdir, 'doc_classes')
class_file_names = [f for f in os.listdir(doc_dir) if f.endswith('.xml')]
file_list += [os.path.join(doc_dir, f) for f in class_file_names]
elif not os.path.isfile(path):
file_list += [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.xml')]
elif os.path.isfile(path) and path.endswith('.xml'):
file_list.append(path)
for cur_file in file_list:
tree = ET.parse(cur_file)
doc = tree.getroot()
if 'version' not in doc.attrib:
print("Version missing from 'doc'")
sys.exit(255)
version = doc.attrib['version']
if doc.attrib['name'] in class_names:
continue
class_names.append(doc.attrib['name'])
classes[doc.attrib['name']] = doc
class_names.sort()
# Don't make class list for Sphinx, :toctree: handles it
# make_class_list(class_names, 2)
for cn in class_names:
c = classes[cn]
make_rst_class(c)