zephyrproject-rtos/zephyr
0
0
Fork 0
mirror of https://github.com/zephyrproject-rtos/zephyr synced 2025-08-03 19:55:22 +00:00
Code Issues Packages Projects Releases Wiki Activity
57a402cd0a
zephyr/doc/scripts/gen_devicetree_rest.py
Carles Cufi 0d02de6113 doc: script: Default to empty string in DT docset env var 
Windows does not default to an empty string when an env var is not
declared, like Linux does. Instead, explicitly default to one in order
for the .strip() call to work properly.

Signed-off-by: Carles Cufi <carles.cufi@nordicsemi.no>
2020-12-01 14:55:19 -05:00

645 lines
21 KiB
Python
Raw Blame History

# Copyright (c) 2020 Nordic Semiconductor ASA
# SPDX-License-Identifier: Apache-2.0
"""
Like gen_kconfig_rest.py, but for generating an index of existing
devicetree bindings.
"""
import argparse
from collections import defaultdict
import glob
import io
import logging
import os
from pathlib import Path
import pprint
import re
import sys
import textwrap
import edtlib
import gen_helpers
# A line of '-' characters in vendor-prefixes.txt that separates
# the data from comments about it.
VENDOR_PREFIXES_SEPARATOR = '-' * 50
ZEPHYR_BASE = Path(__file__).parents[2]
GENERIC_OR_VENDOR_INDEPENDENT = 'Generic or vendor-independent'
UNKNOWN_VENDOR = 'Unknown vendor'
ZEPHYR_BASE = Path(__file__).parents[2]
# Base properties that have documentation in 'dt-important-props'.
DETAILS_IN_IMPORTANT_PROPS = set('compatible label reg status interrupts'.split())
logger = logging.getLogger('gen_devicetree_rest')
class VndLookup:
"""
A convenience class for looking up information based on a
devicetree compatible's vendor prefix 'vnd'.
"""
def __init__(self, vendor_prefixes, bindings):
self.vnd2vendor = self.load_vnd2vendor(vendor_prefixes)
self.vnd2bindings = self.init_vnd2bindings(bindings)
self.vnd2ref_target = self.init_vnd2ref_target()
def vendor(self, vnd):
return self.vnd2vendor.get(vnd, UNKNOWN_VENDOR)
def bindings(self, vnd, default=None):
return self.vnd2bindings.get(vnd, default)
def target(self, vnd):
return self.vnd2ref_target.get(
vnd, self.vnd2ref_target[(UNKNOWN_VENDOR,)])
@staticmethod
def load_vnd2vendor(vendor_prefixes):
# Load the vendor-prefixes.txt file. Return a dict mapping 'vnd'
# vendor prefixes as they are found in compatible properties to
# each vendor's full name.
#
# For example, this line:
#
# vnd A stand-in for a real vendor
#
# Gets split into a key 'vnd' and a value 'A stand-in for a real
# vendor' in the return value.
#
# The 'None' key maps to GENERIC_OR_VENDOR_INDEPENDENT.
vnd2vendor = {
None: GENERIC_OR_VENDOR_INDEPENDENT,
}
found_separator = False # have we found the '-----' separator?
with open(vendor_prefixes, 'r') as f:
for line in f:
line = line.strip()
if line and found_separator:
# Every line after the separator should be in this form:
#
# <vnd><TAB><vendor>
vnd_vendor = line.split('\t', 1)
assert len(vnd_vendor) == 2, line
vnd2vendor[vnd_vendor[0]] = vnd_vendor[1]
elif line.startswith(VENDOR_PREFIXES_SEPARATOR):
found_separator = True
logger.info('found %d vendor prefixes in %s', len(vnd2vendor) - 1,
vendor_prefixes)
if logger.isEnabledFor(logging.DEBUG):
logger.debug('vnd2vendor=%s', pprint.pformat(vnd2vendor))
return vnd2vendor
def init_vnd2bindings(self, bindings):
# Take a 'vnd2vendor' map and a list of bindings and return a dict
# mapping 'vnd' vendor prefixes prefixes to lists of bindings. The
# bindings in each list are sorted by compatible. The keys in the
# return value are sorted by vendor name.
#
# Special cases:
#
# - The 'None' key maps to bindings with no vendor prefix
# in their compatibles, like 'gpio-keys'. This is the first key.
# - The (UNKNOWN_VENDOR,) key maps to bindings whose compatible
# has a vendor prefix that exists, but is not known,
# like 'somethingrandom,device'. This is the last key.
# Get an unsorted dict mapping vendor prefixes to lists of bindings.
unsorted = defaultdict(list)
generic_bindings = []
unknown_vendor_bindings = []
for binding in bindings:
vnd = compatible_vnd(binding.compatible)
if vnd is None:
generic_bindings.append(binding)
elif vnd in self.vnd2vendor:
unsorted[vnd].append(binding)
else:
unknown_vendor_bindings.append(binding)
# Key functions for sorting.
def vnd_key(vnd):
return self.vnd2vendor[vnd].casefold()
def binding_key(binding):
return binding.compatible
# Sort the bindings for each vendor by compatible.
# Plain dicts are sorted in CPython 3.6+, which is what we
# support, so the return dict's keys are in the same
# order as vnd2vendor.
#
# The unknown-vendor bindings being inserted as a 1-tuple key is a
# hack for convenience that ensures they won't collide with a
# known vendor. The code that consumes the dict below handles
# this.
vnd2bindings = {
None: sorted(generic_bindings, key=binding_key)
}
for vnd in sorted(unsorted, key=vnd_key):
vnd2bindings[vnd] = sorted(unsorted[vnd], key=binding_key)
vnd2bindings[(UNKNOWN_VENDOR,)] = sorted(unknown_vendor_bindings,
key=binding_key)
if logger.isEnabledFor(logging.DEBUG):
logger.debug('vnd2bindings: %s', pprint.pformat(vnd2bindings))
return vnd2bindings
def init_vnd2ref_target(self):
# The return value, vnd2ref_target, is a dict mapping vendor
# prefixes to ref targets for their relevant sections in this
# file, with these special cases:
#
# - The None key maps to the ref target for bindings with no
# vendor prefix in their compatibles, like 'gpio-keys'
# - The (UNKNOWN_VENDOR,) key maps to the ref target for bindings
# whose compatible has a vendor prefix that is not recognized.
vnd2ref_target = {}
for vnd in self.vnd2bindings:
if vnd is None:
vnd2ref_target[vnd] = 'dt_no_vendor'
elif isinstance(vnd, str):
vnd2ref_target[vnd] = f'dt_vendor_{vnd}'
else:
assert vnd == (UNKNOWN_VENDOR,), vnd
vnd2ref_target[vnd] = 'dt_unknown_vendor'
return vnd2ref_target
def main():
args = parse_args()
setup_logging(args.verbose)
bindings = load_bindings(args.dts_roots)
base_binding = load_base_binding()
vnd_lookup = VndLookup(args.vendor_prefixes, bindings)
dump_content(bindings, base_binding, vnd_lookup, args.out_dir)
def parse_args():
# Parse command line arguments from sys.argv.
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', default=0, action='count',
help='increase verbosity; may be given multiple times')
parser.add_argument('--vendor-prefixes', required=True,
help='vendor-prefixes.txt file path')
parser.add_argument('--dts-root', dest='dts_roots', action='append',
help='''additional DTS root directory as it would
be set in DTS_ROOTS''')
parser.add_argument('out_dir', help='output files are generated here')
return parser.parse_args()
def setup_logging(verbose):
if verbose >= 2:
log_level = logging.DEBUG
elif verbose == 1:
log_level = logging.INFO
else:
log_level = logging.ERROR
logging.basicConfig(format='%(filename)s:%(levelname)s: %(message)s',
level=log_level)
def load_bindings(dts_roots):
# Get a list of edtlib.Binding objects from searching 'dts_roots'.
if not dts_roots:
sys.exit('no DTS roots; use --dts-root to specify at least one')
binding_files = []
for dts_root in dts_roots:
binding_files.extend(glob.glob(f'{dts_root}/dts/bindings/**/*.yaml'))
bindings = edtlib.bindings_from_paths(binding_files, ignore_errors=True)
num_total = len(bindings)
# Remove bindings from the 'vnd' vendor, which is not a real vendor,
# but rather a stand-in we use for examples and tests when a real
# vendor would be inappropriate.
bindings = [binding for binding in bindings if
compatible_vnd(binding.compatible) != 'vnd']
logger.info('found %d bindings (ignored %d) in this dts_roots list: %s',
len(bindings), num_total - len(bindings), dts_roots)
return bindings
def load_base_binding():
# Make a Binding object for base.yaml.
#
# This helps separate presentation for properties common to all
# nodes from node-specific properties.
base_yaml = ZEPHYR_BASE / 'dts' / 'bindings' / 'base' / 'base.yaml'
if not base_yaml.is_file():
sys.exit(f'Expected to find base.yaml at {base_yaml}')
return edtlib.Binding(os.fspath(base_yaml), {}, require_compatible=False,
require_description=False)
def dump_content(bindings, base_binding, vnd_lookup, out_dir):
# Dump the generated .rst files for a vnd2bindings dict.
# Files are only written if they are changed. Existing .rst
# files which would not be written by the 'vnd2bindings'
# dict are removed.
out_dir = Path(out_dir)
setup_bindings_dir(bindings, out_dir)
write_bindings_rst(vnd_lookup, out_dir)
write_per_compatible_orphans(bindings, base_binding, vnd_lookup, out_dir)
def setup_bindings_dir(bindings, out_dir):
# Make a set of all the Path objects we will be creating for
# out_dir / bindings / {compatible}.rst. Delete all the ones that
# shouldn't be there. Make sure the bindings output directory
# exists.
paths = set()
bindings_dir = out_dir / 'bindings'
logger.info('making output subdirectory %s', bindings_dir)
bindings_dir.mkdir(parents=True, exist_ok=True)
for binding in bindings:
paths.add(bindings_dir / binding_filename(binding))
for path in bindings_dir.iterdir():
if path not in paths:
logger.info('removing unexpected file %s', path)
path.unlink()
def write_bindings_rst(vnd_lookup, out_dir):
# Write out_dir / bindings.rst, the top level index of bindings.
string_io = io.StringIO()
print_block(f'''\
.. _devicetree_binding_index:
Bindings index
##############
This page documents the available devicetree bindings.
See {zref('dt-bindings')} for an introduction to the Zephyr bindings
file format.
Vendor index
************
This section contains an index of hardware vendors.
Click on a vendor's name to go to the list of bindings for
that vendor.
.. rst-class:: rst-columns
''', string_io)
for vnd in vnd_lookup.vnd2bindings:
print(f'- :ref:`{vnd_lookup.target(vnd)}`', file=string_io)
print_block('''\
Bindings by vendor
******************
This section contains available bindings, grouped by vendor.
Within each group, bindings are listed by the "compatible" property
they apply to, like this:
**Vendor name (vendor prefix)**
.. rst-class:: rst-columns
- <compatible-A>
- <compatible-B> (on <bus-name> bus)
- <compatible-C>
- ...
The text "(on <bus-name> bus)" appears when bindings may behave
differently depending on the bus the node appears on.
For example, this applies to some sensor device nodes, which may
appear as children of either I2C or SPI bus nodes.
''', string_io)
for vnd, bindings in vnd_lookup.vnd2bindings.items():
if isinstance(vnd, tuple):
title = vnd[0]
else:
title = vnd_lookup.vendor(vnd).strip()
if isinstance(vnd, str):
title += f' ({vnd})'
underline = '=' * len(title)
print_block(f'''\
.. _{vnd_lookup.target(vnd)}:
{title}
{underline}
.. rst-class:: rst-columns
''', string_io)
for binding in bindings:
print(f'- :ref:`{binding_ref_target(binding)}`', file=string_io)
print(file=string_io)
write_if_updated(out_dir / 'bindings.rst', string_io.getvalue())
def write_per_compatible_orphans(bindings, base_binding, vnd_lookup, out_dir):
# For each compatible in vnd2bindings, write the corresponding
# out_dir / bindings / {binding.compatible}.rst file, if its
# content needs updating. This file is an 'orphan' because it's not
# in any toctree.
logging.info('updating up to %d :orphan: files', len(bindings))
num_written = 0
# Names of properties in base.yaml.
base_names = set(base_binding.prop2specs.keys())
for binding in bindings:
string_io = io.StringIO()
# :orphan:
#
# .. ref_target:
#
# Title [(on <bus> bus)]
# ######################
if binding.on_bus:
on_bus_title = f' (on {binding.on_bus} bus)'
else:
on_bus_title = ''
compatible = binding.compatible
title = f'{compatible}{on_bus_title}'
underline = '#' * len(title)
print_block(f'''\
:orphan:
.. _{binding_ref_target(binding)}:
{title}
{underline}
''', string_io)
# Vendor: <link-to-vendor-section>
vnd = compatible_vnd(compatible)
print('Vendor: '
f':ref:`{vnd_lookup.vendor(vnd)} <{vnd_lookup.target(vnd)}>`\n',
file=string_io)
# Binding description.
if binding.bus:
bus_help = f'These nodes are "{binding.bus}" bus nodes.'
else:
bus_help = ''
print_block(f'''\
Description
***********
{bus_help}
''', string_io)
print(to_code_block(binding.description.strip()), file=string_io)
# Properties.
print_block('''\
Properties
**********
''', string_io)
print_top_level_properties(binding, base_names, string_io)
print_child_binding_properties(binding, string_io)
# Specifier cells.
#
# This presentation isn't particularly nice. Perhaps something
# better can be done for future work.
if binding.specifier2cells:
print_block('''\
Specifier cell names
********************
''', string_io)
for specifier, cells in binding.specifier2cells.items():
print(f'- {specifier} cells: {", ".join(cells)}',
file=string_io)
written = write_if_updated(out_dir / 'bindings' /
binding_filename(binding),
string_io.getvalue())
if written:
num_written += 1
logging.info('done writing :orphan: files; %d files needed updates',
num_written)
def print_top_level_properties(binding, base_names, string_io):
# Print the RST for top level properties for 'binding' to 'string_io'.
#
# The 'base_names' set contains all the base.yaml properties.
def prop_table(filter_fn):
# Get a properly formatted and indented table of properties.
specs = [prop_spec for prop_spec in binding.prop2specs.values()
if filter_fn(prop_spec)]
indent = ' ' * 14
if specs:
temp_io = io.StringIO()
print_property_table(specs, temp_io)
return textwrap.indent(temp_io.getvalue(), indent)
return indent + '(None)'
if binding.child_binding:
print_block('''\
Top level properties
====================
''', string_io)
if binding.prop2specs:
if binding.child_binding:
print_block(f'''
These property descriptions apply to "{binding.compatible}"
nodes themselves. This page also describes child node
properties in the following sections.
''', string_io)
print_block(f'''\
.. tabs::
.. group-tab:: Node specific properties
Properties not inherited from the base binding file.
{prop_table(lambda prop_spec: prop_spec.name not in base_names)}
.. group-tab:: Base properties
Properties inherited from the base binding file, which defines
common properties that may be set on many nodes. Not all of these
may apply to the "{binding.compatible}" compatible.
{prop_table(lambda prop_spec: prop_spec.name in base_names)}
''', string_io)
else:
print('No top-level properties.\n', file=string_io)
def print_child_binding_properties(binding, string_io):
# Prints property tables for all levels of nesting of child
# bindings.
level = 1
child = binding.child_binding
while child is not None:
if level == 1:
level_string = 'Child'
elif level == 2:
level_string = 'Grandchild'
else:
level_string = f'Level {level} child'
if child.prop2specs:
title = f'{level_string} node properties'
underline = '=' * len(title)
print(f'{title}\n{underline}\n', file=string_io)
print_property_table(child.prop2specs.values(), string_io)
child = child.child_binding
level += 1
def print_property_table(prop_specs, string_io):
# Writes a table of properties based on 'prop_specs', an iterable
# of edtlib.PropertySpec objects, to 'string_io'.
# Table header.
print_block('''\
.. list-table::
:widths: 1 1 4
:header-rows: 1
* - Name
- Type
- Details
''', string_io)
def to_prop_table_row(prop_spec):
# Get a multiline string for a PropertySpec table row.
# The description column combines the description field,
# along with things like the default value or enum values.
#
# The property 'description' field from the binding may span
# one or multiple lines. We try to come up with a nice
# presentation for each.
details = ''
raw_prop_descr = prop_spec.description
if raw_prop_descr:
details += to_code_block(raw_prop_descr)
if prop_spec.required:
details += '\n\nThis property is **required**.'
if prop_spec.default:
details += f'\n\nDefault value: ``{prop_spec.default}``'
if prop_spec.const:
details += f'\n\nConstant value: ``{prop_spec.const}``'
elif prop_spec.enum:
details += ('\n\nLegal values: ' +
', '.join(f'``{repr(val)}``' for val in
prop_spec.enum))
if prop_spec.name in DETAILS_IN_IMPORTANT_PROPS:
details += (f'\n\nSee {zref("dt-important-props")} for more '
'information.')
return f"""\
* - ``{prop_spec.name}``
- ``{prop_spec.type}``
- {textwrap.indent(details, ' ' * 7).lstrip()}
"""
# Print each row.
for prop_spec in prop_specs:
print(to_prop_table_row(prop_spec), file=string_io)
def print_block(block, string_io):
# Helper for dedenting and printing a triple-quoted RST block.
# (Just a block of text, not necessarily just a 'code-block'
# directive.)
print(textwrap.dedent(block), file=string_io)
def to_code_block(s, indent=0):
# Converts 's', a string, to an indented rst .. code-block::. The
# 'indent' argument is a leading indent for each line in the code
# block, in spaces.
indent = indent * ' '
return ('.. code-block:: none\n\n' +
textwrap.indent(s, indent + ' ') + '\n')
def compatible_vnd(compatible):
# Get the vendor prefix for a compatible string 'compatible'.
#
# For example, compatible_vnd('foo,device') is 'foo'.
#
# If 'compatible' has no comma (','), None is returned.
if ',' not in compatible:
return None
return compatible.split(',', 1)[0]
def binding_ref_target(binding):
# Return the sphinx ':ref:' target name for a binding.
ret = re.sub('[,-]', '_', binding.compatible)
if binding.on_bus is not None:
ret += f'_{binding.on_bus}'
return ret
def zref(target, text=None):
# Make an appropriate RST :ref:`text <target>` or :ref:`target`
# string to a zephyr documentation ref target 'target', and return
# it.
#
# By default, the bindings docs are in the main Zephyr
# documentation, but this script supports putting them in a
# separate Sphinx doc set. Since we also link to Zephyr
# documentation from the generated content, we have an environment
# variable based escape hatch for putting the target in the zephyr
# doc set.
#
# This relies on intersphinx:
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html
docset = os.environ.get('GEN_DEVICETREE_REST_ZEPHYR_DOCSET', '')
if docset.strip():
target = f'{docset}:{target}'
if text:
return f':ref:`{text} <{target}>`'
return f':ref:`{target}`'
def binding_filename(binding):
if binding.on_bus is None:
return f'{binding.compatible}.rst'
return f'{binding.compatible}-{binding.on_bus}.rst'
def write_if_updated(path, s):
# gen_helpers.write_if_updated() wrapper that handles logging.
written = gen_helpers.write_if_updated(path, s)
logger.debug('%s %s', 'wrote' if written else 'did NOT write', path)
return written
if __name__ == '__main__':
main()
sys.exit(0)
Reference in New Issue View Git Blame Copy Permalink