Initial setup for jupyterbook build

docs/.gitignore

@@ -0,0 +1,5 @@
+/_toc.yml
+/jupyter_execute
+/conf.py
+/sensai
+/build

docs/_config.yml

@@ -0,0 +1,185 @@
+# Book settings
+# Learn more at https://jupyterbook.org/customize/config.html
+
+#######################################################################################
+# A default configuration that will be loaded for all jupyter books
+# Users are expected to override these values in their own `_config.yml` file.
+# This is also the "master list" of all allowed keys and values.
+
+#######################################################################################
+# Book settings
+title                       : Serena Documentation  # The title of the book. Will be placed in the left navbar.
+author                      : Oraios AI & Oraios Software  # The author of the book
+copyright                   : "2025 by Serena contributors"  # Copyright year to be placed in the footer
+logo                        : ../resources/serena-logo.svg   # A path to the book logo
+# Patterns to skip when building the book. Can be glob-style (e.g. "*skip.ipynb")
+exclude_patterns            : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_rst.py', 'docs/create_toc.py']
+# Auto-exclude files not in the toc
+only_build_toc_files        : true
+
+#######################################################################################
+# Execution settings
+execute:
+  # NOTE: Notebooks are not executed, because test_notebooks.py executes them and stores them with outputs in the docs/ folder
+  # NOTE: If changed, repeat below in `nb_execution_mode`.
+  execute_notebooks         : "off"  # Whether to execute notebooks at build time. Must be one of ("auto", "force", "cache", "off")
+  cache                     : ""    # A path to the jupyter cache that will be used to store execution artifacts. Defaults to `_build/.jupyter_cache/`
+  exclude_patterns          : []    # A list of patterns to *skip* in execution (e.g. a notebook that takes a really long time)
+  timeout                   : 1000    # The maximum time (in seconds) each notebook cell is allowed to run.
+  run_in_temp               : false # If `True`, then a temporary directory will be created and used as the command working directory (cwd),
+                                    # otherwise the notebook's parent directory will be the cwd.
+  allow_errors              : true # If `False`, when a code cell raises an error the execution is stopped, otherwise all cells are always run.
+  stderr_output             : show  # One of 'show', 'remove', 'remove-warn', 'warn', 'error', 'severe'
+
+#######################################################################################
+# Parse and render settings
+parse:
+  myst_enable_extensions: # default extensions to enable in the myst parser. See https://myst-parser.readthedocs.io/en/latest/using/syntax-optional.html
+    - amsmath
+    - colon_fence
+    # - deflist
+    - dollarmath
+    # - html_admonition
+    # - html_image
+    - linkify
+    # - replacements
+    # - smartquotes
+    - substitution
+    - tasklist
+  myst_url_schemes: [ mailto, http, https ] # URI schemes that will be recognised as external URLs in Markdown links
+  myst_dmath_double_inline: true  # Allow display math ($$) within an inline context
+
+#######################################################################################
+# HTML-specific settings
+html:
+  #favicon                   : "_static/images/tianshou-favicon.png"  # A path to a favicon image
+  use_edit_page_button      : false  # Whether to add an "edit this page" button to pages. If `true`, repository information in repository: must be filled in
+  use_repository_button     : false  # Whether to add a link to your repository button
+  use_issues_button         : false  # Whether to add an "open an issue" button
+  use_multitoc_numbering    : true   # Continuous numbering across parts/chapters
+  use_darkmode_button       : false
+  extra_footer              : ""
+  google_analytics_id       : ""  # A GA id that can be used to track book views.
+  home_page_in_navbar       : true  # Whether to include your home page in the left Navigation Bar
+  baseurl                   : "https://tianshou.readthedocs.io/en/master/"
+  analytics:
+
+  comments:
+    hypothesis              : false
+    utterances              : false
+  announcement              : "" # A banner announcement at the top of the site.
+
+#######################################################################################
+# LaTeX-specific settings
+latex:
+  latex_engine              : pdflatex  # one of 'pdflatex', 'xelatex' (recommended for unicode), 'luatex', 'platex', 'uplatex'
+  use_jupyterbook_latex     : true # use sphinx-jupyterbook-latex for pdf builds as default
+  targetname                : book.tex
+# Add a bibtex file so that we can create citations
+#bibtex_bibfiles:
+#  - refs.bib
+
+#######################################################################################
+# Launch button settings
+launch_buttons:
+  notebook_interface        : classic  # The interface interactive links will activate ["classic", "jupyterlab"]
+  binderhub_url             : ""  # The URL of the BinderHub (e.g., https://mybinder.org)
+  jupyterhub_url            : ""  # The URL of the JupyterHub (e.g., https://datahub.berkeley.edu)
+  thebe                     : false  # Add a thebe button to pages (requires the repository to run on Binder)
+  colab_url                 : "https://colab.research.google.com"
+
+repository:
+  url                       : https://github.com/thu-ml/tianshou  # The URL to your book's repository
+  path_to_book              : docs  # A path to your book's folder, relative to the repository root.
+  branch                    : master  # Which branch of the repository should be used when creating links
+
+#######################################################################################
+# Advanced and power-user settings
+sphinx:
+  extra_extensions          :
+    - sphinx.ext.autodoc
+    - sphinx.ext.viewcode
+    - sphinx_toolbox.more_autodoc.sourcelink
+    #- sphinxcontrib.spelling
+  local_extensions          :   # A list of local extensions to load by sphinx specified by "name: path" items
+  recursive_update          : false # A boolean indicating whether to overwrite the Sphinx config (true) or recursively update (false)
+  config                    :   # key-value pairs to directly over-ride the Sphinx configuration
+    autodoc_typehints_format: "short"
+    autodoc_member_order: "bysource"
+    autoclass_content: "both"
+    autodoc_mock_imports:
+     - numpy
+     - pandas
+     - torch
+     - scipy
+     - sklearn
+     - seaborn
+     - matplotlib
+     - pytorch_lightning
+     - torchtext
+     - tensorboard
+     - lightgbm
+     - sklearn
+     - seaborn
+     - psutil
+     - pyyaml
+     - sqlite3
+     - azureml
+     - mlflow
+     - MySQLdb
+     - catboost
+     - xgboost
+     - clearml
+     - azure
+     - geopandas
+     - shapely
+     - networkx
+     - utm
+     - sklearn_quantile
+    autodoc_default_options:
+      show-inheritance: True
+    html_js_files:
+#     We have to list them explicitly because they need to be loaded in a specific order
+      - js/vega@5.js
+      - js/vega-lite@5.js
+      - js/vega-embed@5.js
+    autodoc_show_sourcelink: True
+    add_module_names: False
+    github_username: thu-ml
+    github_repository: tianshou
+    nb_execution_mode: "off"
+    nb_merge_streams: True  # This is important for cell outputs to appear as single blocks rather than one block per line
+    python_use_unqualified_type_names: True
+    nb_mime_priority_overrides: [
+      [ 'html', 'application/vnd.jupyter.widget-view+json', 10 ],
+      [ 'html', 'application/javascript', 20 ],
+      [ 'html', 'text/html', 30 ],
+      [ 'html', 'text/latex', 40 ],
+      [ 'html', 'image/svg+xml', 50 ],
+      [ 'html', 'image/png', 60 ],
+      [ 'html', 'image/jpeg', 70 ],
+      [ 'html', 'text/markdown', 80 ],
+      [ 'html', 'text/plain', 90 ],
+      [ 'spelling', 'application/vnd.jupyter.widget-view+json', 10 ],
+      [ 'spelling', 'application/javascript', 20 ],
+      [ 'spelling', 'text/html', 30 ],
+      [ 'spelling', 'text/latex', 40 ],
+      [ 'spelling', 'image/svg+xml', 50 ],
+      [ 'spelling', 'image/png', 60 ],
+      [ 'spelling', 'image/jpeg', 70 ],
+      [ 'spelling', 'text/markdown', 80 ],
+      [ 'spelling', 'text/plain', 90 ],
+    ]
+    mathjax_path: https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js
+    mathjax3_config:
+      loader: { load: [ '[tex]/configmacros' ] }
+      tex:
+        packages: { '[+]': [ 'configmacros' ] }
+        macros:
+          vect: ["{\\mathbf{\\boldsymbol{#1}} }", 1]
+          E: "{\\mathbb{E}}"
+          P: "{\\mathbb{P}}"
+          R: "{\\mathbb{R}}"
+          abs: ["{\\left| #1 \\right|}", 1]
+          simpl: ["{\\Delta^{#1} }", 1]
+          amax: "{\\text{argmax}}"

docs/autogen_rst.py

@@ -0,0 +1,155 @@
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Optional, List
+
+log = logging.getLogger(os.path.basename(__file__))
+
+TOP_LEVEL_PACKAGE = "sensai"
+PROJECT_NAME = "sensAI"
+
+def module_template(module_qualname: str):
+    module_name = module_qualname.split(".")[-1]
+    title = module_name.replace("_", r"\_")
+    return f"""{title}
+{"=" * len(title)}
+
+.. automodule:: {module_qualname}
+   :members:
+   :undoc-members:
+"""
+
+
+def index_template(package_name: str, doc_references: Optional[List[str]] = None, text_prefix=""):
+    doc_references = doc_references or ""
+    if doc_references:
+        doc_references = "\n" + "\n".join(f"* :doc:`{ref}`" for ref in doc_references) + "\n"
+
+    dirname = package_name.split(".")[-1]
+    title = dirname.replace("_", r"\_")
+    if title == TOP_LEVEL_PACKAGE:
+        title = "API Reference"
+    return f"{title}\n{'=' * len(title)}" + text_prefix + doc_references
+
+
+def write_to_file(content: str, path: str):
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w") as f:
+        f.write(content)
+    os.chmod(path, 0o666)
+
+
+_SUBTITLE = (
+    f"\n Here is the autogenerated documentation of the {PROJECT_NAME} API. \n \n "
+    "The Table of Contents to the left has the same structure as the "
+    "repository's package code. The links at each page point to the submodules and subpackages. \n"
+)
+
+
+def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix=""):
+    """Creates/updates documentation in form of rst files for modules and packages.
+
+    Does not delete any existing rst files. Thus, rst files for packages or modules that have been removed or renamed
+    should be deleted by hand.
+
+    This method should be executed from the project's top-level directory
+
+    :param src_root: path to library base directory, typically "src/<library_name>"
+    :param rst_root: path to the root directory to which .rst files will be written
+    :param clean: whether to completely clean the target directory beforehand, removing any existing .rst files
+    :param overwrite: whether to overwrite existing rst files. This should be used with caution as it will delete
+        all manual changes to documentation files
+    :package_prefix: a prefix to prepend to each module (for the case where the src_root is not the base package),
+        which, if not empty, should end with a "."
+    :return:
+    """
+    rst_root = os.path.abspath(rst_root)
+
+    if clean and os.path.isdir(rst_root):
+        shutil.rmtree(rst_root)
+
+    base_package_name = package_prefix + os.path.basename(src_root)
+
+    # TODO: reduce duplication with same logic for subpackages below
+    files_in_dir = os.listdir(src_root)
+    module_names = [f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_")]
+    subdir_refs = [
+        f"{f}/index"
+        for f in files_in_dir
+        if os.path.isdir(os.path.join(src_root, f))
+        and not f.startswith("_")
+        and not f.startswith(".")
+    ]
+    package_index_rst_path = os.path.join(
+        rst_root,
+        "index.rst",
+    )
+    log.info(f"Writing {package_index_rst_path}")
+    write_to_file(
+        index_template(
+            base_package_name,
+            doc_references=module_names + subdir_refs,
+            text_prefix=_SUBTITLE,
+        ),
+        package_index_rst_path,
+    )
+
+    for root, dirnames, filenames in os.walk(src_root):
+        if os.path.basename(root).startswith("_"):
+            continue
+        base_package_relpath = os.path.relpath(root, start=src_root)
+        base_package_qualname = package_prefix + os.path.relpath(
+            root,
+            start=os.path.dirname(src_root),
+        ).replace(os.path.sep, ".")
+
+        for dirname in dirnames:
+            if dirname.startswith("_"):
+                log.debug(f"Skipping {dirname}")
+                continue
+            files_in_dir = os.listdir(os.path.join(root, dirname))
+            module_names = [
+                f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_")
+            ]
+            subdir_refs = [
+                f"{f}/index"
+                for f in files_in_dir
+                if os.path.isdir(os.path.join(root, dirname, f)) and not f.startswith("_")
+            ]
+            package_qualname = f"{base_package_qualname}.{dirname}"
+            package_index_rst_path = os.path.join(
+                rst_root,
+                base_package_relpath,
+                dirname,
+                "index.rst",
+            )
+            log.info(f"Writing {package_index_rst_path}")
+            write_to_file(
+                index_template(package_qualname, doc_references=module_names + subdir_refs),
+                package_index_rst_path,
+            )
+
+        for filename in filenames:
+            base_name, ext = os.path.splitext(filename)
+            if ext == ".py" and not filename.startswith("_"):
+                module_qualname = f"{base_package_qualname}.{filename[:-3]}"
+
+                module_rst_path = os.path.join(rst_root, base_package_relpath, f"{base_name}.rst")
+                if os.path.exists(module_rst_path) and not overwrite:
+                    log.debug(f"{module_rst_path} already exists, skipping it")
+
+                log.info(f"Writing module documentation to {module_rst_path}")
+                write_to_file(module_template(module_qualname), module_rst_path)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    docs_root = Path(__file__).parent
+    enable_module_docs = False
+    if enable_module_docs:
+        make_rst(
+            docs_root / ".." / "src" / "serena",
+            docs_root / "serena",
+            clean=True,
+        )

docs/create_toc.py

@@ -0,0 +1,8 @@
+import os
+from pathlib import Path
+
+# This script provides a platform-independent way of making the jupyter-book call (used in pyproject.toml)
+toc_file = Path(__file__).parent / "_toc.yml"
+cmd = f"jupyter-book toc from-project docs -e .rst -e .md -e .ipynb  >{toc_file}"
+print(cmd)
+os.system(cmd)

docs/custom_agent.md

@@ -63,5 +63,5 @@ configuration file (`.yml`).
 It should be straightforward to incorporate Serena into any
 agent framework (like [pydantic-ai](https://ai.pydantic.dev/), [langgraph](https://langchain-ai.github.io/langgraph/tutorials/introduction/) or others).
 Typically, you need only to write an adapter for Serena's tools to the tool representation in the framework of your choice, 
-as was done by us for Agno with [SerenaAgnoToolkit](/src/serena/agno.py).
+as was done by us for Agno with `SerenaAgnoToolkit` (see `/src/serena/agno.py`).
 

pyproject.toml

@@ -65,6 +65,12 @@ dev = [
   "types-pyyaml>=6.0.12.20241230",
   "syrupy>=4.9.1",
   "types-requests>=2.32.4.20241230",
+  # docs
+  "sphinx==5.0.2",
+  "sphinx_rtd_theme>=0.5.1",
+  "sphinx-toolbox==3.7.0",
+  "jupyter-book==0.15.1",
+  "nbsphinx"
 ]
 agno = ["agno>=2.2.1", "sqlalchemy>=2.0.40"]
 google = ["google-genai>=1.8.0"]
@@ -123,6 +129,14 @@ lint = ["_black_check", "_ruff_check"]
 format = ["_ruff_format", "_black_format"]
 _mypy = "mypy src/serena"
 type-check = ["_mypy"]
+# docs
+_autogen_rst = "python docs/autogen_rst.py"
+_sphinx_build = "sphinx-build -b html docs docs/_build -W --keep-going"
+_jb_generate_toc = "python docs/create_toc.py"
+_jb_generate_config = "jupyter-book config sphinx docs/"
+doc-clean = "rm -rf docs/_build docs/03_api"
+doc-generate-files = ["_autogen_rst", "_jb_generate_toc", "_jb_generate_config"]
+doc-build = ["doc-clean", "doc-generate-files", "_sphinx_build"]
 
 [tool.ruff]
 target-version = "py311"