aboutsummaryrefslogtreecommitdiff
path: root/utils/python/doc_gen
diff options
context:
space:
mode:
Diffstat (limited to 'utils/python/doc_gen')
-rw-r--r--utils/python/doc_gen/__init__.py0
-rw-r--r--utils/python/doc_gen/doc_gen.py93
-rw-r--r--utils/python/doc_gen/doxygen_extractor.py242
-rw-r--r--utils/python/doc_gen/md_converter.py242
-rw-r--r--utils/python/doc_gen/system_utils.py137
5 files changed, 714 insertions, 0 deletions
diff --git a/utils/python/doc_gen/__init__.py b/utils/python/doc_gen/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/utils/python/doc_gen/__init__.py
diff --git a/utils/python/doc_gen/doc_gen.py b/utils/python/doc_gen/doc_gen.py
new file mode 100644
index 0000000..97d6e65
--- /dev/null
+++ b/utils/python/doc_gen/doc_gen.py
@@ -0,0 +1,93 @@
+import os, re, json, xml.etree.ElementTree
+from optparse import OptionParser
+
+from doxygen_extractor import DoxygenExtractor
+from md_converter import MarkdownConverter
+from system_utils import SystemUtils
+
+member_func_filter = ["idleCallback", "systemCallback", "~"]
+
+filters = True
+
+utils = SystemUtils()
+
+###
+# the trigger for generating our documentation
+###
+def generate_mkdocs(header_paths, type_colour = "#a71d5d", function_name_colour = "#795da3"):
+
+ global member_func_filter
+ doxygen = DoxygenExtractor(os.path.abspath("."), header_paths)
+ markdown = MarkdownConverter(type_colour, function_name_colour, separate_defaults = True, display_defaults = False)
+
+ doxygen.generate_doxygen()
+ #utils.validate_version(doxygen.working_dir, header_paths, "./docs/archive")
+
+ file_names = utils.find_files('docs','*.md')
+ section_kind = ["public-func"]
+ meta_data_regex = re.compile( r'\[comment\]: <> \((.*?)\)', re.MULTILINE | re.DOTALL )
+
+ for filename in file_names:
+ print(filename)
+
+ read_lines = utils.read(filename)
+
+ file_lines = markdown.clean(read_lines, meta_data_regex)
+
+ utils.write(filename, file_lines)
+
+ previous = ""
+
+ for line_number, line in enumerate(file_lines, 1):
+
+ result = re.findall(meta_data_regex,line)
+
+ if len(result) is not 0:
+
+ meta_data = json.loads(result[0])
+
+ if previous is not "" and "end" in meta_data.keys() and meta_data['end'] == previous:
+ previous = ""
+ continue
+ elif previous is "":
+ try:
+ previous = meta_data['className']
+ except:
+ raise Exception('There isn\'t a match for the meta_data '+ meta_data)
+ else:
+ raise Exception('There isn\'t a match for the meta_data \''+ previous + "'")
+
+ local_filter = member_func_filter
+
+ if "filter" in meta_data:
+ for member_function in meta_data["filter"]:
+ local_filter = local_filter + [ str(member_function) ]
+
+ print "Custom filter applied: " + str(member_func_filter)
+
+ class_xml_files = list(utils.find_files("./xml","*class*"+meta_data['className'] + ".xml"))
+
+ print class_xml_files
+
+ if len(class_xml_files) == 0:
+ raise Exception("Invalid classname: " + meta_data['className'])
+ elif len(class_xml_files) > 1:
+ class_xml_files
+
+ doxygen_class_xml = xml.etree.ElementTree.parse(class_xml_files[0]).getroot()
+
+ member_functions = []
+
+ for section_def in doxygen_class_xml.iter('sectiondef'):
+ if section_def.attrib['kind'] in section_kind:
+ for member_func in section_def.iter('memberdef'):
+ new_member = doxygen.extract_member_function(member_func, local_filter, filter= filters)
+ if new_member is not None:
+ member_functions.append(new_member)
+
+ before = file_lines[:line_number]
+ after = file_lines[line_number:]
+
+ between = markdown.gen_member_func_doc(meta_data['className'], member_functions)
+
+ utils.write(filename, before + between + after)
diff --git a/utils/python/doc_gen/doxygen_extractor.py b/utils/python/doc_gen/doxygen_extractor.py
new file mode 100644
index 0000000..9207c9d
--- /dev/null
+++ b/utils/python/doc_gen/doxygen_extractor.py
@@ -0,0 +1,242 @@
+import os
+from system_utils import SystemUtils
+
+class DoxygenExtractor:
+
+ md_special_chars =[
+ {
+ "md_char": "*",
+ "replacement": "&#42;"
+ },
+ {
+ "md_char": "#",
+ "replacement": "&#35;"
+ },
+ {
+ "md_char": "`",
+ "replacement": "&#183;"
+ }
+ ]
+
+ #constructor
+ def __init__(self, root, header_paths, working_dir = "./temp", doxygen_xml_dest = "./xml"):
+ os.chdir(root)
+ self.header_paths = header_paths
+ self.utils = SystemUtils()
+ self.doxygen_xml_dest = doxygen_xml_dest
+ self.working_dir = working_dir
+
+ ###
+ # this function copies headers recursively from a source director to a destination
+ # directory.
+ ###
+ def get_headers(self, from_dir, to_dir):
+ self.utils.copy_files(from_dir, to_dir, "*.h")
+
+ ###
+ # Strips out reserved characters used in markdown notation, and replaces them
+ # with html character codes.
+ #
+ # @param text the text to strip and replace the md special characters
+ #
+ # @return the stripped text.
+ ###
+ def escape_md_chars(self, text):
+ for char in self.md_special_chars:
+ text = text.replace(char['md_char'], "\\" + char['md_char'])
+ return text
+
+
+ ###
+ # this function extracts data from an element tag ignoring the tag 'ref', but
+ # obtains the textual data it has inside the ref tag.
+ #
+ # @param element the element to process
+ #
+ # @return a list of extracted strings.
+ ###
+ def extract_ignoring_refs(self, element):
+ list = []
+
+ if element.text is not None:
+ list.append(element.text)
+
+ for ref in element.iter(tag="ref"):
+ list.append(ref.text)
+
+ return list
+
+ ###
+ # this function extracts data from an element tag including all sub elements
+ # (recursive)
+ #
+ # @param element the element to process
+ #
+ # @return a list of extracted strings.
+ ###
+ def extract_with_subelements(self, element):
+ list = []
+
+ list.append(element.text or "")
+
+ #if element.text is not None:
+ #list.append(element.text)
+
+ for subelement in element:
+ if subelement is not None:
+ list = list + self.extract_with_subelements(subelement)
+
+ list.append(element.tail or "")
+
+ return list
+
+ ###
+ # this function was at one point intended to fetch a value of a default parameter
+ # it is now only used to fetch the default parameters' name.
+ #
+ # @param document_root the root of the entire document
+ # @param element the element containing the default parameter
+ #
+ # @return a dictionary containing:
+ # {
+ # 'name':'',
+ # 'value':''
+ # }
+ #
+ # @note this would be more useful if it return the value, it currently does not.
+ ###
+ def extract_default(self, element):
+ ref = element.find("ref")
+ return {'name':' '.join(element.itertext()), 'value':''}
+
+ ###
+ # extracts a member function form the xml document
+ #
+ # @param root the document root
+ # @param xml_element the member function xml element.
+ #
+ # @return a function dictionary:
+ # {
+ # 'short_name':"",
+ # 'name':"",
+ # 'return_type':"",
+ # 'params':[],
+ # 'description':[],
+ # 'returns':"",
+ # 'notes':"",
+ # 'examples':""
+ # }
+ ###
+ def extract_member_function(self, xml_element, function_filter = [], filter = True):
+
+ function = {
+ 'short_name':"",
+ 'name':"",
+ 'return_type':"",
+ 'params':[],
+ 'description':[],
+ 'returns':"",
+ 'notes':"",
+ 'examples':""
+ }
+
+ function['name'] = xml_element.find('definition').text
+ function['short_name'] = xml_element.find('name').text
+
+ if filter and any(filtered_func in function['short_name'] for filtered_func in function_filter):
+ print "Filtered out: " + function['short_name']
+ return
+
+ print "Generating documentation for: " + function['short_name']
+
+ if xml_element.find('type') is not None:
+ function['return_type'] = self.escape_md_chars(' '.join(self.extract_ignoring_refs(xml_element.find('type'))))
+
+ #extract our parameters for this member function
+ for parameter in xml_element.iter('param'):
+
+ type = ""
+ name = ""
+
+ if parameter.find('type') is not None:
+ type = self.escape_md_chars(' '.join(parameter.find('type').itertext()))
+
+ if parameter.find('declname') is not None:
+ name = ' '.join(self.extract_ignoring_refs(parameter.find('declname')))
+
+ param_object = {
+ 'type': type,
+ 'name': name,
+ 'default':{
+ 'name':"",
+ 'value':""
+ }
+ }
+
+ if parameter.find('defval') is not None:
+ extracted = self.extract_default(parameter.find('defval'))
+ param_object['default']['name'] = extracted['name']
+ param_object['default']['value'] = extracted['value']
+
+ function['params'].append(param_object)
+
+
+ detailed_description = xml_element.find('detaileddescription')
+
+ if len(detailed_description.findall("para")) is not 0:
+ for para in detailed_description.findall("para"):
+ if len(para.findall("programlisting")) is 0 and len(para.findall("simplesect")) is 0:
+ function['description'] = function['description'] + self.extract_with_subelements(para)
+
+ #para indicates a new paragraph - we should treat it as such... append \n!
+ function['description'] = function['description'] + ["\n\n"]
+
+ if len(detailed_description.findall("para/simplesect[@kind='return']/para")) is not 0:
+ return_section = detailed_description.findall("para/simplesect[@kind='return']/para")[0]
+ function['returns'] = ' '.join(return_section.itertext())
+
+ if len(detailed_description.findall("para/simplesect[@kind='note']/para")) is not 0:
+ return_section = detailed_description.findall("para/simplesect[@kind='note']/para")[0]
+ function['notes'] = ' '.join(return_section.itertext())
+
+ examples = detailed_description.find('para/programlisting')
+
+ if examples is not None:
+ function['examples'] = ''.join([('' if index is 0 else ' ')+word for index, word in enumerate(examples.itertext(),1) ])
+
+ param_list = detailed_description.findall('para/parameterlist')
+
+ if len(param_list) is not 0:
+ for parameter_desc in param_list[0].findall('parameteritem'):
+
+ param_descriptor = {
+ 'name':'',
+ 'description':''
+ }
+
+ param_name = parameter_desc.findall('parameternamelist/parametername')
+ additional = parameter_desc.findall('parameterdescription/para')
+
+ if len(param_name) is not 0:
+ param_descriptor['name'] = param_name[0].text
+
+ if len(additional) is not 0:
+ param_descriptor['description'] = ' '.join(additional[0].itertext())
+
+ for descriptor in function['params']:
+ if param_descriptor['name'] in descriptor['name']:
+ descriptor['description'] = param_descriptor['description']
+
+ return function
+
+ def generate_doxygen(self):
+ self.utils.mk_dir(self.working_dir)
+ self.utils.clean_dir(self.working_dir)
+
+ for path in self.header_paths:
+ self.get_headers(path, self.working_dir)
+
+ if os.path.exists(self.doxygen_xml_dest):
+ self.utils.clean_dir(self.doxygen_xml_dest)
+
+ os.system('doxygen doxy-config.cfg')
diff --git a/utils/python/doc_gen/md_converter.py b/utils/python/doc_gen/md_converter.py
new file mode 100644
index 0000000..ff3a1eb
--- /dev/null
+++ b/utils/python/doc_gen/md_converter.py
@@ -0,0 +1,242 @@
+import re, json, copy
+
+class MarkdownConverter:
+
+ #constructor
+ def __init__(self, type_colour, function_name_colour, separate_defaults = True, display_defaults = False):
+ self.type_colour = type_colour
+ self.function_name_colour = function_name_colour
+ self.separate_defaults = separate_defaults
+ self.display_defaults = display_defaults
+
+ ###
+ # wraps text in a div element with a given color
+ #
+ # @param text the text to wrap
+ # @param color the desired text color
+ #
+ # @return a string representing the now wrapped text
+ ###
+ def wrap_text(self, text, color):
+ return "<div style='color:" + color + "; display:inline-block'>" + text + "</div>"
+
+ ###
+ # removes previously generated markdown from the file.
+ #
+ # @param file_lines a list of lines representing a file.
+ # @param regexp the regular expression that dictates a match.
+ ###
+ def clean(self, file_lines, regexp):
+ start = 0
+ end = 0
+
+ for line_number, line in enumerate(file_lines, 1):
+ result = re.findall(regexp,line)
+
+ if len(result) is not 0:
+ meta_data = json.loads(result[0])
+
+ keys = meta_data.keys()
+
+ #classname indicates the beginning of a meta_data section
+ if 'className' in keys:
+ start = line_number
+
+ #end indicated the end of a meta_data section
+ if 'end' in keys:
+ end = line_number - 1
+
+ return file_lines[:start] + file_lines[end:]
+
+ ###
+ # given a member function, this function derives the alternative versions
+ #
+ # @param member_func the member function that is required to be derrived
+ #
+ # @return a list of function dictionaries that contain the alternatives, based on the original
+ ###
+ def derive_functions(self, member_func):
+ member_functions_derived = []
+
+ if len(member_func['params']) is not 0:
+
+ param_index = 0
+
+ for param in member_func['params']:
+ if len(param['default']['name']) is 0:
+ param_index = param_index + 1
+ else:
+ break
+
+ bare_function = {
+ 'short_name' : member_func['short_name'],
+ 'name' : member_func['name'],
+ 'params' : [],
+ 'description' : member_func['description'],
+ 'returns' : member_func['returns'],
+ 'notes' : member_func['notes'],
+ 'examples' : member_func['examples'],
+ 'return_type' : member_func['return_type'],
+ }
+
+ for i in range(0, param_index):
+ bare_function['params'] = bare_function['params'] + [member_func['params'][i]]
+
+ member_functions_derived = member_functions_derived + [bare_function]
+
+ current = copy.copy(bare_function)
+
+ #lists retain references, so we have to copy objects to maintain separation
+ for remainder in range(param_index, len(member_func['params'])):
+ current['params'] = current['params'] + [member_func['params'][remainder]]
+ member_functions_derived = member_functions_derived + [current]
+ current = copy.copy(current)
+
+ else:
+ member_functions_derived = member_functions_derived + [member_func]
+
+ return member_functions_derived
+
+ ###
+ # given a parameter, this function generates text
+ #
+ # @param param the parameter that needs a textual translation
+ #
+ # @return a string representing the parameter
+ ###
+ def gen_param_text(self, param):
+ text = "\n> "
+
+ if param['type'] is not None:
+ text = text + " " + self.wrap_text(param['type'], self.type_colour)
+
+ text = text + " " + param['name']
+
+ if self.display_defaults:
+ if len(param['default']['name']) is not 0:
+ text = text + " `= " + param['default']['name']
+
+ if len(param['default']['value']) is not 0:
+ text = text + param['default']['value']
+
+ text = text + "`"
+
+ if 'description' in param.keys():
+ text = text +" - " + param['description']
+
+ text = text.encode('ascii','ignore')
+
+ return text
+
+ ###
+ # given a list of member functions, this function returns a list of new lines for the
+ # file currently being processed.
+ #
+ # @param class_name the name of the current class (found in the meta data)
+ # @param member_functions the list of member_functions extracted from XML
+ #
+ # @return a list containing the new lines to be inserted into the current file.
+ ###
+ def gen_member_func_doc(self, class_name, member_functions):
+
+ # this is what a member function dictionary contains.
+ # function = {
+ # 'short_name':"",
+ # 'name':"",
+ # 'return_type':"",
+ # 'params':[],
+ # 'description':[],
+ # 'returns':"",
+ # 'notes':"",
+ # 'examples':"",
+ # 'default':None
+ # }
+
+ lines = []
+
+ for index, member_func in enumerate(member_functions,0):
+
+ member_functions_derived = []
+
+ if index is 0 or member_func['short_name'] != member_functions[index - 1]['short_name']:
+ if class_name == member_func["short_name"]:
+ lines.append("##Constructor\n")
+ else:
+ lines.append("##" + member_func["short_name"]+"\n")
+
+ #we want to clearly separate our different level of functions in the DAL
+ #so we present methods with defaults as overloads.
+ if self.separate_defaults is True:
+ member_functions_derived = member_functions_derived + self.derive_functions(member_func)
+
+ for derived_func in member_functions_derived:
+ #---- short name for urls ----
+ lines.append("<br/>\n")
+
+ short_name = ""
+
+ if len(derived_func["return_type"]) is not 0:
+ short_name = "####" + self.wrap_text(derived_func["return_type"],self.type_colour) + " " +self.wrap_text(derived_func["short_name"], self.function_name_colour) + "("
+ else:
+ short_name = "####" + derived_func["short_name"] + "("
+
+ last_param = None
+
+ if len(derived_func['params']) is not 0:
+ last_param = derived_func['params'][-1]
+
+ #generate parameters for the name of this function
+ for param in derived_func['params']:
+ text = ""
+
+ if param['type'] is not None:
+ text = text + " " + self.wrap_text(param['type'], self.type_colour)
+
+ text = text + " " + param['name']
+
+ if param is not last_param:
+ short_name = short_name + text +", "
+ else:
+ short_name = short_name + text
+
+ lines.append(short_name + ")\n")
+ #-----------------------------
+
+ #---- description ----
+ if len(derived_func['description']) is not 0:
+ lines.append("#####Description\n")
+ lines.append(' '.join(derived_func['description']) + "\n")
+ #-----------------------------
+
+ #---- parameters ----
+ if len(derived_func['params']) is not 0:
+ lines.append("#####Parameters\n")
+
+ for param in derived_func['params']:
+ lines.append(self.gen_param_text(param) + "\n")
+ #-----------------------------
+
+ #---- returns ----
+ if len(derived_func['returns']) is not 0:
+ lines.append("#####Returns\n")
+ lines.append(derived_func['returns'] + "\n")
+ #-----------------------------
+
+ #---- examples ----
+ if len(derived_func['examples']) is not 0:
+ lines.append("#####Example\n")
+ lines.append("```cpp\n")
+ lines.append(derived_func['examples'])
+ lines.append("```\n")
+ #-----------------------------
+
+ #---- notes ----
+ if len(derived_func['notes']) is not 0:
+ lines.append("\n!!! note\n")
+ lines.append(" " + derived_func['notes'].replace('\n','\n '))
+ lines.append('\n\n')
+ #-----------------------------
+
+ lines.append("____\n")
+
+ return lines
diff --git a/utils/python/doc_gen/system_utils.py b/utils/python/doc_gen/system_utils.py
new file mode 100644
index 0000000..326eb21
--- /dev/null
+++ b/utils/python/doc_gen/system_utils.py
@@ -0,0 +1,137 @@
+import json, shutil, zipfile, urllib, os, fnmatch
+
+class SystemUtils:
+
+ folder_filter = ["ble", "ble-nrf51822", "mbed-classic","nrf51-sdk"]
+
+ ###
+ # reads a file and returns a list of lines
+ #
+ # @param path the path where the file is located
+ #
+ # @return the list of lines representing the file.
+ ###
+ def read(self, path, plain=False):
+ if plain:
+ return self.__read_plain(path)
+ print "Opening: " + path + " \n"
+ with open(path, 'r') as file:
+ return file.readlines()
+
+ def __read_plain(self, path):
+ print "Opening: " + path + " \n"
+ with open(path, 'r') as file:
+ return file.read()
+
+ ###
+ # writes a given set of lines to a path.
+ #
+ # @param path the path where the file is located
+ # @param lines the lines to write
+ ###
+ def write(self, path, lines):
+ print "Writing to: " + path + " \n"
+ with open(path, 'w') as file:
+ file.writelines(lines)
+
+ #http://stackoverflow.com/questions/2186525/use-a-glob-to-find-files-recursively-in-python
+ def find_files(self, directory, pattern):
+
+ print("DIR:")
+ for root, dirs, files in os.walk(directory):
+ if any(dir in root for dir in self.folder_filter):
+ continue
+
+ for basename in files:
+ if fnmatch.fnmatch(basename, pattern):
+ filename = os.path.join(root, basename)
+ yield filename
+
+ ###
+ # removes files from a folder.
+ ###
+ def clean_dir(self, dir):
+ for root, dirs, files in os.walk(dir):
+ for f in files:
+ os.unlink(os.path.join(root, f))
+ for d in dirs:
+ shutil.rmtree(os.path.join(root, d))
+
+ ###
+ # this files from one location to another
+ ###
+ def copy_files(self, from_dir, to_dir, pattern):
+
+
+ files = self.find_files(from_dir, pattern)
+
+ print("FILES!!!! ")
+ for file in files:
+ print file
+ shutil.copy(file,to_dir)
+
+ def mk_dir(self, path):
+ if not os.path.exists(path):
+ os.makedirs(path)
+
+ def copytree(self, src, dst, symlinks=False, ignore=None):
+ if not os.path.exists(dst):
+ os.makedirs(dst)
+ for item in os.listdir(src):
+ s = os.path.join(src, item)
+ d = os.path.join(dst, item)
+ if os.path.isdir(s):
+ self.copytree(s, d, symlinks, ignore)
+ else:
+ if not os.path.exists(d) or os.stat(s).st_mtime - os.stat(d).st_mtime > 1:
+ shutil.copy2(s, d)
+
+ def __add_version_info(self,version_string, extract_location):
+ content_path = extract_location + "js/base.js"
+ lines = self.read(content_path)
+ html_string = '<div class=\'admonition warning\' style=\'margin-top:30px;\'><p class=\'admonition-title\'>Warning</p><p>You are viewing documentation for <b>' + version_string + '</b></p></div>'
+ lines[0]= '$(document).ready(function() { $(\'div[role="main"]\').prepend("' + html_string + '") });'
+ self.write(content_path, lines)
+
+ def validate_version(self, working_dir, module_paths, extract_location):
+ import yaml
+
+ module_string = "/module.json"
+ mkdocs_yml = yaml.load(self.read("./mkdocs.yml", plain=True))
+
+ module_strings = []
+
+ for current_path in module_paths:
+ module_strings = module_strings + [json.loads(self.read(current_path + module_string, plain=True))["version"]]
+
+ if module_strings[1:] != module_strings[:-1]:
+ raise Exception("Version mismatch exception! microbit-dal and microbit are not compatible versions.")
+
+ module_string = "v" + str(module_strings[0])
+
+ if mkdocs_yml["versioning"]["runtime"] != module_string:
+ #capture old site, save in docs/historic/versionNumber
+ zip_dest = working_dir + "/" + str(mkdocs_yml["versioning"]["runtime"]) + ".zip"
+
+ extract_folder = extract_location+ "/" + mkdocs_yml["versioning"]["runtime"]+"/"
+
+ urllib.urlretrieve("https://github.com/lancaster-university/microbit-docs/archive/gh-pages.zip", zip_dest)
+
+ zip_ref = zipfile.ZipFile(zip_dest)
+
+ #obtain the archive prepended name
+ archive_name = working_dir + "/" + zip_ref.namelist()[0]
+
+ zip_ref.extractall(working_dir)
+ zip_ref.close()
+
+ self.copytree(archive_name, extract_folder)
+
+ self.__add_version_info(mkdocs_yml["versioning"]["runtime"], extract_folder)
+
+ self.clean_dir(archive_name)
+
+ mkdocs_yml["versioning"]["runtime"] = module_string
+
+ with open("./mkdocs.yml", "w") as f:
+ yaml.dump(mkdocs_yml, f, default_flow_style=False )
generated by cgit v1.2.3 (git 2.39.1) at 2025-06-27 23:36:57 +0000