Files
gtav-src/tools_ng/techart/dcc/motionbuilder2014/python/RS/Docs/RSTConverter.py
T
2025-09-29 00:52:08 +02:00

630 lines
23 KiB
Python
Executable File

"""
Methods for reading python files and converting their content into a friendly reStructuredText format so that
Sphinx can generate proper documentation.
Authors:
David Vega <david.vega@rockstargames.com>
Image:
$TechArt\Tools\test.png
"""
import os
import re
import sys
import inspect
from importlib import import_module
from collections import OrderedDict
from xml.etree import cElementTree as xml
from snakefood import find
import logging
from RS import Config, Perforce
from RS.Utils.Path import ResolvePath
HEADER = ( ".. raw:: html\n\n"
"\t<table>\n"
"\t\t<tr>\n"
"\t\t\t<td><img src={imagePath}><td/>\n"
"\t\t\t<td>\n"
"\t\t\t{string}\n"
"\t\t\t<td/>\n"
"\t\t</tr>\n"
"\t</table>\n\n")
AUTHORS = (".. topic:: Authors & Contributors\n\n"
"{string}\n\n")
EXAMPLE = ("**Examples:**\n"
"::\n\n"
"{string}\n\n")
ARGUMENTS = "{string}"
RETURN = ":return:{string}\n"
IMAGE = "Image : {string}"
END = "---///---"
TYPES = {"HEADER": ["Description"],
"AUTHORS": ["Author", "Authors"],
"EXAMPLE": ["Usage", "Example", "Useage"],
"ARGUMENTS": ["Argument", "Arguments", "Args", "Kwarg", "Keyword argument", "Keyword arguments", "Kwargs"],
"RETURN": ["Return", "Returns"],
"IMAGE": ["Image", "Icon"],
"END": ["End"]}
TYPE_VALUES = {value[index].capitalize(): key for key, value in TYPES.iteritems() for index in xrange(len(value)) }
HAS_PARAMETER_TYPE = re.compile("(?<=\(|:)[a-zA-Z0-9_. ]*(?=\)|;)").search
# Create a search function to find each type of header type
# Create the regular expression string from the types dictionary
SEARCH_STRING = []
[SEARCH_STRING.extend(value) for value in TYPES.values()]
SEARCH_STRING = "".join("{}|".format(string) for string in SEARCH_STRING)
SEARCH_STRING = SEARCH_STRING[:-1]
# Match string from the end and if it must have at least two characters between the whitespaces
SEARCH_REGEX = re.compile("(?P<type>{}){}".format(SEARCH_STRING, "[a-z0-9 ]{0,2}:$"), re.I).search
# For some reason, sphinx does not recognize /n when it comes to showing the doc strings of modules/methods/classes etc.
SPHINX_NEWLINE = " " * 80
# Resolves the proper directory for the image module without calling RS.Config
DEFAULT_IMAGE_PATH = os.path.join(os.path.abspath(Config.Script.Path.ToolImages), "Tools", "default.png")
def GetAllPythonFiles(rootDirectory):
"""
Gets all the python files within a directory
Arguments:
rootDirectory (string): path to the directory with the python files
Returns:
list[string, etc.]
"""
return [os.path.join(path, eachFile) for path, folders, files in os.walk(rootDirectory) for eachFile in files
if eachFile.endswith(".py") and os.path.isfile(os.path.join(path, eachFile))]
def CreateDirectory(module, root=None, pathSections=()):
"""
Creates a directory that matches the current directory hierarchy of the module within the top level package this
module belongs to.
Arguments:
module (module): python module to create directory for
root (string): directory to add new directories to
pathSections (list[string, etc.]): additional directories to add between the root directory and module
Return:
(string): path to the new directory
"""
# Get the root directory (RS)
nameParts = module.__name__.split(".")
filepath = module.__file__
moduleRoot, filename = os.path.split(filepath)
if not root:
root = moduleRoot
# We do this to get the root RS folder it belongs to
while not root.endswith(nameParts[0]):
filepath = root
root, filename = os.path.split(filepath)
folderPath = os.path.join(root, *pathSections)
for part in nameParts[1:-1]:
folderPath = os.path.join(folderPath, part)
if not os.path.isdir(folderPath):
os.mkdir(folderPath)
return root, folderPath
def CreateIndexFile(rootDirectory):
"""
Generates an index file that contains links to rst file locations for each python file in the provided directory
Arguments:
rootDirectory (string): path to the directory with the python files
"""
rootPath = os.path.split(__file__)[0]
with open(os.path.join(rootPath, "_templates", "index.rst"), "r") as indexFile:
content = indexFile.read()
content = content.format(TableOfContent=FormatTableOfContent(rootDirectory))
Perforce.Edit(os.path.join(rootPath, "RST", "index.rst"))
with open(os.path.join(rootPath, "RST", "index.rst"), "w") as indexFile:
indexFile.write(content)
def CreateRstFile(module):
"""
Creates an RST File for a given module
Arguments:
module (module): module that needs an RST file
"""
root, folderPath = CreateDirectory(module, pathSections=("Docs", "RST"))
moduleShortName = module.__name__.split(".")[-1]
# Open the template Module RST File
with open(os.path.join(root, "Docs", "_templates", "module.rst"), "r") as rstFile:
rstContent = rstFile.read()
# 13 is for adding some extra padding to the title line
content = rstContent.format(moduleFullName=module.__name__, moduleName=moduleShortName,
moduleDocString=FormatModuleDocString(module),
titleLine="*" * (len(module.__name__) + len(moduleShortName) + 13))
rstPath = os.path.join(folderPath, "{}.rst".format(moduleShortName))
with open(rstPath, "w") as rstFile:
rstFile.write(content)
def CreateCacheFile(module):
"""
Stores the doc information of a module in an xml to avoid importing it in the future
Arguments:
module (module): module that needs its doc string cached
"""
# Get the root directory (RS)
root, folderPath = CreateDirectory(module, pathSections=("Docs", "_cache"))
moduleShortName = module.__name__.split(".")[-1]
xmlPath = os.path.join(folderPath, "{}.xml".format(moduleShortName))
tree = None
treeRoot = xml.Element("module")
treeRoot.attrib["name"] = module.__name__
treeRoot.attrib["image"] = GetImagePath(module.__doc__)
treeRoot.attrib["path"] = module.__file__
treeRoot.attrib["revision"] = "0"
treeRoot.attrib["changelist"] = "0"
treeRoot.attrib["changelistDescription"] = ""
treeRoot.attrib["date"] = "0/0/0 0:00:00"
authorsElement, importElement, descriptionElement, methodsElement, classesElement = AddElements(treeRoot,
("authors", "imports", "description", "methods", "classes"))
descriptionElement.text = module.__doc__
perforceData = Perforce.Run("files", ["-a", module.__file__])
if perforceData is not None and perforceData.Records:
treeRoot.attrib["revision"] = str(perforceData.Records[0]["rev"])
treeRoot.attrib["changelist"] = str(perforceData.Records[0]["change"])
treeRoot.attrib["date"] = Perforce.ConvertPerforceSecondsToDate(int(perforceData.Records[0]["time"]))
for index, records in enumerate(perforceData.Records):
description = Perforce.GetDescription(records["change"])
if not index:
treeRoot.attrib["changelistDescription"] = str(description["desc"])
if authorsElement.find("author[@username='{}']".format(description["user"])) is None:
user = Perforce.GetUserData(description["user"])
if not user:
continue
moduleElement = xml.Element("author")
moduleElement.attrib["name"] = user.get("FullName", "")
moduleElement.attrib["username"] = user.get("User", "")
moduleElement.attrib["email"] = user.get("Email", "")
authorsElement.append(moduleElement)
for modules in find.find_imports(module.__file__, None, []):
moduleElement = xml.Element("import")
moduleElement.attrib["name"] = modules[0]
importElement.append(moduleElement)
for name, member in inspect.getmembers(module):
if not (hasattr(member, "__module__") and member.__module__ == module.__name__):
continue
if inspect.isfunction(member):
element = AddElements(methodsElement, ["method[@name='{}']".format(name)])[0]
element.attrib['name'] = name
element.attrib["image"] = GetImagePath(member.__doc__)
element.text = member.__doc__
elif inspect.isclass(member):
classElement = AddElements(classesElement, ["class[@name='{}']".format(name)])[0]
classMethodsElement = AddElements(classElement, ["methods"])[0]
classElement.attrib['name'] = name
classElement.attrib["image"] = GetImagePath(member.__doc__)
classElement.text = member.__doc__
for methodname, classmember in member.__dict__.iteritems():
if type(classmember).__name__ == "function":
element = AddElements(classMethodsElement, ["method[@name='{}']".format(methodname)])[0]
element.attrib['name'] = methodname
element.attrib["image"] = GetImagePath(member.__doc__)
element.text = classmember.__doc__
indent(treeRoot)
tree = tree or xml.ElementTree(treeRoot)
tree.write(xmlPath)
def GetImagePath(docString):
"""
Returns the image path found in the docstring
Arguments:
docString (string): doc string for module, class or method
Return:
string
"""
current = HEADER
stringList = []
imagePath = DEFAULT_IMAGE_PATH
if docString is not None:
docStringList = docString.splitlines() + ["END:"]
for eachLine in docStringList:
eachLine = eachLine.lstrip()
searchResult = SEARCH_REGEX(eachLine)
if searchResult:
_current = getattr(sys.modules[__name__],
TYPE_VALUES[searchResult.group("type").capitalize()])
if current != _current and current == IMAGE:
imagePath = re.sub(r"[\\\\]+", r"\\", "".join(stringList).strip().encode("string_escape"))
break
current = _current
continue
if current == IMAGE:
stringList.append(eachLine)
return imagePath
def AddElements(element, elementNames=()):
"""
Looks for child elements within the passed element and creates them if they don't exist
Arguments:
element (xml.etree.cElementTree.Element): parent xml element
elementNames (string): name of the child elements to find/create
Return:
list[xml.etree.cElementTree.Element, etc.]
"""
elements = []
for index, elementName in enumerate(elementNames):
childElement = element.find(elementName)
if childElement is None:
elementName = elementName.split("[")[0]
childElement = xml.Element(elementName)
element.insert(index, childElement)
elements.append(childElement)
return elements
def CreateRstFiles(rootDirectory, progressbar=None):
"""
Dynamically creates RST Files for the RS Tool base
Arguments:
rootDirectory (string): path of the directory with python files that need to have RST files created.
progressbar (RS.Tools.UI.ExternalProgressBar): progressbar to update
"""
progress = 0
package_path = os.path.split(__file__)[0]
log = os.path.join(package_path, "doc.log")
logging.basicConfig(filename=log, level=logging.ERROR, format='%(levelname)s %(asctime)s : %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
parentDirectory, rootName = os.path.split(rootDirectory)
for path, folders, files in os.walk(rootDirectory):
for eachFile in files:
filepath = os.path.join(path, eachFile)
baseName, extension = os.path.splitext(eachFile)
if ".py" == extension and os.path.isfile(filepath) and "batchbuild" not in baseName.lower():
if progressbar:
progress += 1
progressbar.Update(value=progress, text="Caching {}".format(eachFile))
name = filepath.replace(parentDirectory, "")
name = os.path.splitext(name)[0].replace("\\", ".")[1:]
try:
module = import_module(name)
logging.debug("--- {} RST file successfully made!".format(name))
except Exception, error:
logging.error("!!! {} RST file unsuccesful!, Error: {}".format(name, error))
continue
CreateCacheFile(module)
CreateRstFile(module)
def DocStringParser(stringList, imagePath=r"X:\wildwest\dcc\motionbuilder2014\images\Tools\default.png", tabLevel=1,
moduleName=""):
"""
Goes through the provided list of strings and customizes them for the help documentation
Arguments:
stringList: list[string, etc.]; list of strings that make up a doc string
imagePath: path to the image associated with the module
Return:
list[string, etc.]
"""
current = HEADER
convertedDocString = []
strings = []
toolImage = None
stringList = list(stringList)
stringList.append("END:")
cachedDocPath = os.path.join(Config.Script.Path.RockstarRoot, "Docs", "_cache", *moduleName.split(".")[1:])
if os.path.isdir(cachedDocPath):
cachedDocPath = os.path.join(cachedDocPath, "__init__.xml")
else:
cachedDocPath = "{}.xml".format(cachedDocPath)
tree = xml.parse(cachedDocPath)
root = tree.getroot()
authors = ["* {} <{}>".format(contributor.attrib.get("name"), contributor.attrib.get("email"))
for contributor in (root.find("authors") or [])]
for eachLine in stringList:
searchResult = SEARCH_REGEX(eachLine)
if searchResult:
next = getattr(sys.modules[__name__],
TYPE_VALUES[searchResult.group("type").capitalize()])
if current != next:
string = "".join(strings)
# Remove Description Line from Header
if current == HEADER:
string = string.replace("Description:", "")
string = string.strip()
string = string.replace("\n", "<br>\n")
elif current == AUTHORS:
string = "".join(["{}{}\n".format("\t"*tabLevel, author) for author in authors])
authorFlag = True
elif current == ARGUMENTS:
pass
elif current == IMAGE:
toolImage = str(ResolvePath(string.strip()))
toolImage = toolImage.encode("string_escape")
toolImage = re.sub(r"[\\]+", r"\\", toolImage)
strings = []
current = next
continue
if current == HEADER:
convertedDocString[0:0] = [current.format(string=string, imagePath="{imagePath}")]
else:
convertedDocString.append(current.format(string=string, imagePath="{imagePath}"))
strings = []
current = next
continue
# Add bullet to author names and emails
if current == AUTHORS and eachLine.strip():
#eachLine = "*{}".format(eachLine)
continue
if current == ARGUMENTS and (":" in eachLine or "=" in eachLine):
parameter, content = re.split(":|=", eachLine.strip(), 1)
parameterType = ""
for each in (parameter, content):
typeSearch = HAS_PARAMETER_TYPE(each)
if typeSearch:
result = typeSearch.group().strip()
parameterType = "{} ".format(result)
each[:] = each.replace(result, "")
break
eachLine = "\n:param {parameterType}{parameter}: {content}".format(parameter=parameter,
parameterType=parameterType,
content=content)
strings.append("{}{}\n".format("\t"*tabLevel, eachLine))
# Add the image path to the
imagePath = toolImage or imagePath
try:
descriptionString = convertedDocString[0].format(imagePath=imagePath)
convertedDocString[0] = descriptionString
authorFlag = False
for eachLine in convertedDocString:
if eachLine.startswith(".. topic:: Authors & Contributors"):
authorFlag = True
break
if not authorFlag:
string = "".join(["{}{}\n".format("\t"*tabLevel, author) for author in authors])
try:
convertedDocString[1:1] = AUTHORS.format(string=string, imagePath="{imagePath}")
except:
convertedDocString.append(AUTHORS.format(string=string, imagePath="{imagePath}"))
except Exception, error:
print error
# convertedDocString.append(current.format(string="".join(strings), imagePath=imagePath))
return convertedDocString
def FormatModuleDocString(module):
"""
Formats the module doc string for injecting it into the RST file
Arguments:
module (module): module that contains a doc string
Return:
string
"""
docstring = module.__doc__
if not docstring:
return ""
imagePath = r"X:\wildwest\dcc\motionbuilder2014\images\Tools\default.png"
convertedDocString = DocStringParser(docstring.splitlines(), imagePath=imagePath, moduleName=module.__name__)
return "".join(convertedDocString)
def FormatTableOfContent(rootDirectory):
"""
Creates the toctree structure for the index.rst page
Arguments:
rootDirectory (string): path to the directory with the python files
Return:
list[string, etc.]
"""
content = OrderedDict()
contentFormat = "\t{} <{}>\n".format
for path, folders, files in os.walk(rootDirectory):
partialPath = path.replace(rootDirectory, "")
folderpath = partialPath.replace("\\", "/")
packageName = "RS{}".format(folderpath.replace("/", "."))
for eachFile in files:
baseName, extension = os.path.splitext(eachFile)
if not extension.endswith(".py"):
continue
else:
filepath = "".join((folderpath[1:], "/", baseName))
content.setdefault(packageName, []).append(contentFormat(baseName, filepath))
stringList = []
for key, value in content.iteritems():
content[key][0:0] = [key, "\n", "=" * len(key), "\n", "\n", ".. toctree::\n",
"\n"]
content[key].append("\n")
stringList.extend(value)
return "".join(stringList)
def FormatMethodDocString(stringList, imagePath=r"X:\wildwest\dcc\motionbuilder2014\images\Tools\default.png",
tabLevel=1):
"""
Goes through the provided list of strings and customizes them for the help documentation.
Editing the content of the doc strings by intercepting the lines being sent to Sphinx does not give the same results
as editing an RST File directly. So how a doc string is formatted has to be different.
Arguments:
stringList: list[string, etc.]; list of strings that make up a doc string
imagePath: string; path to image associated with the method/class
Return:
list[string, etc.]
"""
stringList = list(stringList)
stringList.append("END:")
current = HEADER
convertedDocString = []
strings = []
returnStrings = []
for eachLine in stringList:
# Search if one of the headers (Description, Arguments, Return, etc.) have been found
searchResult = SEARCH_REGEX(eachLine)
if searchResult:
_current = getattr(sys.modules[__name__],
TYPE_VALUES[searchResult.group().split(":")[0].strip().capitalize()])
if current != _current:
convertedDocString.append(SPHINX_NEWLINE)
convertedDocString.extend(strings)
strings = []
current = _current
continue
if current == ARGUMENTS and (":" in eachLine or "=" in eachLine):
splitLine = re.split(":|=", eachLine.strip(), 1)
parameterType = ""
for index, each in enumerate(splitLine):
typeSearch = HAS_PARAMETER_TYPE(each)
if typeSearch and not index:
parameterType = "{} ".format(typeSearch.group().strip())
splitLine[index] = each.split("(")[0].strip()
break
parameter, content = splitLine
eachLine = ":param {parameterType}{parameter}: {content}".format(parameter=parameter.strip(),
parameterType=parameterType,
content=content.strip())
elif current == IMAGE and eachLine.strip():
imagePath = str(ResolvePath(eachLine.strip()))
imagePath = imagePath.encode("string_escape")
imagePath = re.sub(r"[\\]+", r"\\", imagePath)
continue
elif current == RETURN:
returnStrings.append(eachLine)
continue
strings.append("{}{}\n".format("\t"*tabLevel, eachLine))
convertedDocString.extend(strings)
if convertedDocString:
convertedDocString[0] = convertedDocString[0].strip()
convertedDocString[0:0] = [".. image:: {}\n".format(imagePath), SPHINX_NEWLINE, "\n"]
if returnStrings:
convertedDocString.append(RETURN.format(string="".join(returnStrings)))
return convertedDocString
def indent(elem, level=0):
i = "\n" + level*" "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i