AI Engine (AIE) Backend support as external plugin (#1390)

* Add initial plugin support for external backends (e.g. aie4ml)

* pre-commit fixes

* remove unused JSON include

* pre-commit fix

* make plugin loader consistent with hls4ml

* simplify plugin loader

Author: dimdano

docs/advanced/plugins.rst

@@ -0,0 +1,60 @@
+=======================================
+External Backend and Writer Plugins
+=======================================
+
+``hls4ml`` can discover and load backend implementations from
+external Python packages. This enables specialised flows, such as the AMD AIE backend, to live in
+independent projects that version and iterate at their own cadence while reusing the core
+conversion infrastructure.
+
+Discovery
+=========
+
+Plugin packages advertise themselves through the ``hls4ml.backends`` Python entry point group. Each
+entry exposes a callable that receives ``register_backend`` and ``register_writer`` helpers and performs any setup that is
+required. ``hls4ml`` automatically scans for these entry points during ``hls4ml.backends`` import so
+third-party backends become available without additional user configuration.
+
+In addition to entry points, modules listed in the ``HLS4ML_BACKEND_PLUGINS`` environment variable
+are imported and treated as registration callables. The variable accepts an ``os.pathsep`` separated
+list (``:`` on Linux/macOS or ``;`` on Windows):
+
+.. code-block:: bash
+
+   export HLS4ML_BACKEND_PLUGINS=aie4ml.plugin:another_pkg.hls4ml_backend
+
+Authoring a Plugin
+==================
+
+A minimal plugin registers both a backend and an accompanying writer. The example below
+shows how the ``aie4ml`` package exposes its backend via ``pyproject.toml`` and a ``register``
+function:
+
+.. code-block:: toml
+
+   [project.entry-points."hls4ml.backends"]
+   AIE = "aie4ml.plugin:register"
+
+.. code-block:: python
+
+   # aie4ml/plugin.py
+   from aie4ml.aie_backend import AIEBackend
+   from aie4ml.writer import AIEWriter
+
+   def register(*, register_backend, register_writer):
+       register_writer('AIE', AIEWriter)
+       register_backend('AIE', AIEBackend)
+
+When the plugin is installed, ``hls4ml.backends.get_available_backends()`` will report the new
+backend just like the built-in FPGA toolflows.
+
+Packaging Data Files
+====================
+
+Backends often rely on firmware templates or device description files. These assets should be
+packaged alongside the Python sources using the usual ``setuptools`` mechanisms (``package-data`` or
+``include-package-data``) so they are available from the installed distribution.
+
+For an end-to-end example see the companion ``aie4ml`` [https://github.com/dimdano/aie4ml] package that ships alongside this project
+as a standalone distribution; it encapsulates the existing AMD AIE backend as an installable plugin
+depending on ``hls4ml``.

docs/index.rst

@@ -53,6 +53,7 @@
     advanced/extension
     advanced/model_optimization
     advanced/bramfactor
+    advanced/plugins
 
 .. toctree::
     :hidden:

hls4ml/backends/__init__.py

@@ -1,6 +1,7 @@
 from hls4ml.backends.backend import Backend, get_available_backends, get_backend, register_backend  # noqa: F401
 from hls4ml.backends.fpga.fpga_backend import FPGABackend  # noqa: F401
 from hls4ml.backends.oneapi.oneapi_backend import OneAPIBackend
+from hls4ml.backends.plugin_loader import load_backend_plugins
 from hls4ml.backends.quartus.quartus_backend import QuartusBackend
 from hls4ml.backends.symbolic.symbolic_backend import SymbolicExpressionBackend
 from hls4ml.backends.vivado.vivado_backend import VivadoBackend
@@ -11,10 +12,16 @@
 
 from hls4ml.backends.vitis.vitis_backend import VitisBackend  # isort: skip
 
-register_backend('Vivado', VivadoBackend)
-register_backend('VivadoAccelerator', VivadoAcceleratorBackend)
-register_backend('Vitis', VitisBackend)
-register_backend('Quartus', QuartusBackend)
-register_backend('Catapult', CatapultBackend)
-register_backend('SymbolicExpression', SymbolicExpressionBackend)
-register_backend('oneAPI', OneAPIBackend)
+
+def _register_builtin_backends():
+    register_backend('Vivado', VivadoBackend)
+    register_backend('VivadoAccelerator', VivadoAcceleratorBackend)
+    register_backend('Vitis', VitisBackend)
+    register_backend('Quartus', QuartusBackend)
+    register_backend('Catapult', CatapultBackend)
+    register_backend('SymbolicExpression', SymbolicExpressionBackend)
+    register_backend('oneAPI', OneAPIBackend)
+
+
+_register_builtin_backends()
+load_backend_plugins()

hls4ml/backends/plugin_loader.py

@@ -0,0 +1,86 @@
+"""Utilities for discovering and loading external hls4ml backend plugins."""
+
+import os
+from collections.abc import Callable
+from importlib import import_module
+from importlib.metadata import entry_points
+from typing import Any
+
+from hls4ml.backends.backend import register_backend
+from hls4ml.writer.writers import register_writer
+
+ENTRY_POINT_GROUP = 'hls4ml.backends'
+ENV_PLUGIN_MODULES = 'HLS4ML_BACKEND_PLUGINS'
+
+_plugins_loaded = False
+
+
+def load_backend_plugins() -> None:
+    """Discover and register backend plugins.
+
+    This function loads plugins published via Python entry points under the
+    ``hls4ml.backends`` group as well as modules listed in the
+    ``HLS4ML_BACKEND_PLUGINS`` environment variable. The environment variable
+    accepts a separator compatible with :data:`os.pathsep`.
+    """
+    global _plugins_loaded
+    if _plugins_loaded:
+        return
+
+    _load_entry_point_plugins()
+    _load_env_plugins()
+
+    _plugins_loaded = True
+
+
+def _load_entry_point_plugins() -> None:
+    group_eps = entry_points().select(group=ENTRY_POINT_GROUP)
+
+    for ep in group_eps:
+        try:
+            obj = ep.load()
+        except Exception as exc:
+            print(f'WARNING: failed to load backend plugin entry "{ep.name}": {exc}')
+            continue
+        _register_plugin_object(ep.name, obj)
+
+
+def _load_env_plugins() -> None:
+    raw_modules = os.environ.get(ENV_PLUGIN_MODULES, '')
+    if not raw_modules:
+        return
+
+    for module_name in filter(None, raw_modules.split(os.pathsep)):
+        try:
+            module = import_module(module_name)
+        except Exception as exc:
+            print(f'WARNING: failed to import backend plugin module "{module_name}": {exc}')
+            continue
+
+        register_callable: Any = getattr(module, 'register', module)
+        _register_plugin_object(module_name, register_callable)
+
+
+def _register_plugin_object(name: str, obj: Any) -> None:
+    """Interpret the plugin object and register provided backends."""
+    if callable(obj):
+        _invoke_registration_callable(name, obj)
+        return
+
+    print(f'WARNING: plugin entry "{name}" did not provide a usable backend registration (got {obj!r})')
+
+
+def _invoke_registration_callable(name: str, func: Callable[..., Any]) -> None:
+    try:
+        func(register_backend=register_backend, register_writer=register_writer)
+        return
+    except TypeError:
+        try:
+            func(register_backend, register_writer)
+            return
+        except Exception as exc:
+            print(f'WARNING: backend plugin callable "{name}" failed: {exc}')
+            return
+    except Exception as exc:
+        print(f'WARNING: backend plugin callable "{name}" failed: {exc}')
+        return