From 0aaeaef64cfdce1be1663f5b2e02f0c39e8aa070 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 27 Jul 2025 15:16:57 +0200 Subject: [PATCH 001/220] Updated mcp dependency --- uv.lock | 130 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/uv.lock b/uv.lock index 52fe9264..4322e294 100644 --- a/uv.lock +++ b/uv.lock @@ -4,7 +4,7 @@ requires-python = "==3.11.*" [[package]] name = "agno" -version = "1.7.2" +version = "1.7.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docstring-parser" }, @@ -20,9 +20,9 @@ dependencies = [ { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/50/64/6a82d2375145fd733009db0ada531b595fc2f3c1712ac91c91b2e5fcf02f/agno-1.7.2.tar.gz", hash = "sha256:f4f97e5ea1f9c0781d9bb8396862c82671381386c1569cdb456d70f6c8d3a108", size = 661788, upload-time = "2025-07-10T15:50:06.217Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/84/d2390249d1b0150971a749df79cf653a631227b748fe10be1ef4f426ed2d/agno-1.7.6.tar.gz", hash = "sha256:8df575f86eac6511996045fb90d281ca61fd19774dccbb82f9dc34bda46effdf", size = 724233, upload-time = "2025-07-24T14:36:27.356Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/24/884cbd1133e3b9985d53733250251098caead365bb8706af8da105813abe/agno-1.7.2-py3-none-any.whl", hash = "sha256:37a1985e67762f0d8016318ba5d8b8697c855827d7c86b21a41cdff088294cc2", size = 888229, upload-time = "2025-07-10T15:50:04.262Z" }, + { url = "https://files.pythonhosted.org/packages/d9/7e/5f85c92a34f41f8ffb1467768bae50506f734320c478ed24c4237dc342ea/agno-1.7.6-py3-none-any.whl", hash = "sha256:c6350e44747b37048d4ac080ffa5fb0e0ce2b3d5bc55d37120711ae8d31f27c6", size = 962755, upload-time = "2025-07-24T14:36:25.648Z" }, ] [[package]] @@ -36,7 +36,7 @@ wheels = [ [[package]] name = "anthropic" -version = "0.57.1" +version = "0.59.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -47,9 +47,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d7/75/6261a1a8d92aed47e27d2fcfb3a411af73b1435e6ae1186da02b760565d0/anthropic-0.57.1.tar.gz", hash = "sha256:7815dd92245a70d21f65f356f33fc80c5072eada87fb49437767ea2918b2c4b0", size = 423775, upload-time = "2025-07-03T16:57:35.932Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/52daff015f5a1f24eec891b3041f5f816712fea8b5113dc76638bcbc23d8/anthropic-0.59.0.tar.gz", hash = "sha256:d710d1ef0547ebbb64b03f219e44ba078e83fc83752b96a9b22e9726b523fd8f", size = 425679, upload-time = "2025-07-23T16:23:16.901Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/cf/ca0ba77805aec6171629a8b665c7dc224dab374539c3d27005b5d8c100a0/anthropic-0.57.1-py3-none-any.whl", hash = "sha256:33afc1f395af207d07ff1bffc0a3d1caac53c371793792569c5d2f09283ea306", size = 292779, upload-time = "2025-07-03T16:57:34.636Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b1/03f680393eac04afd8f2be44ee0e39e033c40faf43dbc1c11764b07a2687/anthropic-0.59.0-py3-none-any.whl", hash = "sha256:cbc8b3dccef66ad6435c4fa1d317e5ebb092399a4b88b33a09dc4bf3944c3183", size = 293057, upload-time = "2025-07-23T16:23:14.934Z" }, ] [[package]] @@ -130,11 +130,11 @@ wheels = [ [[package]] name = "certifi" -version = "2025.7.9" +version = "2025.7.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/8a/c729b6b60c66a38f590c4e774decc4b2ec7b0576be8f1aa984a53ffa812a/certifi-2025.7.9.tar.gz", hash = "sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079", size = 160386, upload-time = "2025-07-09T02:13:58.874Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b3/76/52c535bcebe74590f296d6c77c86dabf761c41980e1347a2422e4aa2ae41/certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995", size = 163981, upload-time = "2025-07-14T03:29:28.449Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/66/f3/80a3f974c8b535d394ff960a11ac20368e06b736da395b551a49ce950cce/certifi-2025.7.9-py3-none-any.whl", hash = "sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39", size = 159230, upload-time = "2025-07-09T02:13:57.007Z" }, + { url = "https://files.pythonhosted.org/packages/4f/52/34c6cf5bb9285074dc3531c437b3919e825d976fde097a7a73f79e726d03/certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2", size = 162722, upload-time = "2025-07-14T03:29:26.863Z" }, ] [[package]] @@ -200,11 +200,11 @@ wheels = [ [[package]] name = "docstring-parser" -version = "0.16" +version = "0.17.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/08/12/9c22a58c0b1e29271051222d8906257616da84135af9ed167c9e28f85cb3/docstring_parser-0.16.tar.gz", hash = "sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e", size = 26565, upload-time = "2024-03-15T10:39:44.419Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/9d/c3b43da9515bd270df0f80548d9944e389870713cc1fe2b8fb35fe2bcefd/docstring_parser-0.17.0.tar.gz", hash = "sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912", size = 27442, upload-time = "2025-07-21T07:35:01.868Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/7c/e9fcff7623954d86bdc17782036cbf715ecab1bec4847c008557affe1ca8/docstring_parser-0.16-py3-none-any.whl", hash = "sha256:bf0a1387354d3691d102edef7ec124f219ef639982d096e26e3b60aeffa90637", size = 36533, upload-time = "2024-03-15T10:39:41.527Z" }, + { url = "https://files.pythonhosted.org/packages/55/e2/2537ebcff11c1ee1ff17d8d0b6f4db75873e3b0fb32c2d4a2ee31ecb310a/docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708", size = 36896, upload-time = "2025-07-21T07:35:00.684Z" }, ] [[package]] @@ -267,14 +267,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.44" +version = "3.1.45" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c0/89/37df0b71473153574a5cdef8f242de422a0f5d26d7a9e231e6f169b4ad14/gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269", size = 214196, upload-time = "2025-01-02T07:32:43.59Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110", size = 207599, upload-time = "2025-01-02T07:32:40.731Z" }, + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, ] [[package]] @@ -293,7 +293,7 @@ wheels = [ [[package]] name = "google-genai" -version = "1.25.0" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -305,9 +305,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/59/c9b9148c8702b60253f5a251f16ae436534c5d4362da193c9db05ac9858c/google_genai-1.25.0.tar.gz", hash = "sha256:a08a79c819a5d949d9948cd372e36e512bf85cd28158994daaa36d0ec4cb2b02", size = 228141, upload-time = "2025-07-09T20:53:47.885Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/37/6c0ececc3a7a629029b5beed2ceb9f28f73292236eb96272355636769b0d/google_genai-1.27.0.tar.gz", hash = "sha256:15a13ffe7b3938da50b9ab77204664d82122617256f55b5ce403d593848ef635", size = 220099, upload-time = "2025-07-23T22:00:46.145Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/ec/149f3d49b56cf848142071772aabb1c290b535bd9b5327a5dfccf1d00332/google_genai-1.25.0-py3-none-any.whl", hash = "sha256:fb5cee79b9a0a1b2afd5cfdf279099ecebd186551eefcaa6ec0c6016244e6138", size = 226847, upload-time = "2025-07-09T20:53:46.532Z" }, + { url = "https://files.pythonhosted.org/packages/5a/12/279afe7357af73f9737a3412b6f0bc1482075b896340eb46a2f9cb0fd791/google_genai-1.27.0-py3-none-any.whl", hash = "sha256:afd6b4efaf8ec1d20a6e6657d768b68d998d60007c6e220e9024e23c913c1833", size = 218489, upload-time = "2025-07-23T22:00:44.879Z" }, ] [[package]] @@ -489,7 +489,7 @@ wheels = [ [[package]] name = "jsonschema" -version = "4.24.0" +version = "4.25.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -497,9 +497,9 @@ dependencies = [ { name = "referencing" }, { name = "rpds-py" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/d3/1cf5326b923a53515d8f3a2cd442e6d7e94fcc444716e879ea70a0ce3177/jsonschema-4.24.0.tar.gz", hash = "sha256:0b4e8069eb12aedfa881333004bccaec24ecef5a8a6a4b6df142b2cc9599d196", size = 353480, upload-time = "2025-05-26T18:48:10.459Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/00/a297a868e9d0784450faa7365c2172a7d6110c763e30ba861867c32ae6a9/jsonschema-4.25.0.tar.gz", hash = "sha256:e63acf5c11762c0e6672ffb61482bdf57f0876684d8d249c0fe2d730d48bc55f", size = 356830, upload-time = "2025-07-18T15:39:45.11Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/3d/023389198f69c722d039351050738d6755376c8fd343e91dc493ea485905/jsonschema-4.24.0-py3-none-any.whl", hash = "sha256:a462455f19f5faf404a7902952b6f0e3ce868f3ee09a359b05eca6673bd8412d", size = 88709, upload-time = "2025-05-26T18:48:08.417Z" }, + { url = "https://files.pythonhosted.org/packages/fe/54/c86cd8e011fe98803d7e382fd67c0df5ceab8d2b7ad8c5a81524f791551c/jsonschema-4.25.0-py3-none-any.whl", hash = "sha256:24c2e8da302de79c8b9382fee3e76b355e44d2a4364bb207159ce10b517bd716", size = 89184, upload-time = "2025-07-18T15:39:42.956Z" }, ] [[package]] @@ -558,7 +558,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.11.0" +version = "1.12.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -573,9 +573,9 @@ dependencies = [ { name = "starlette" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/f5/9506eb5578d5bbe9819ee8ba3198d0ad0e2fbe3bab8b257e4131ceb7dfb6/mcp-1.11.0.tar.gz", hash = "sha256:49a213df56bb9472ff83b3132a4825f5c8f5b120a90246f08b0dac6bedac44c8", size = 406907, upload-time = "2025-07-10T16:41:09.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/85/f36d538b1286b7758f35c1b69d93f2719d2df90c01bd074eadd35f6afc35/mcp-1.12.2.tar.gz", hash = "sha256:a4b7c742c50ce6ed6d6a6c096cca0e3893f5aecc89a59ed06d47c4e6ba41edcc", size = 426202, upload-time = "2025-07-24T18:29:05.175Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/92/9c/c9ca79f9c512e4113a5d07043013110bb3369fc7770040c61378c7fbcf70/mcp-1.11.0-py3-none-any.whl", hash = "sha256:58deac37f7483e4b338524b98bc949b7c2b7c33d978f5fafab5bde041c5e2595", size = 155880, upload-time = "2025-07-10T16:41:07.935Z" }, + { url = "https://files.pythonhosted.org/packages/2f/cf/3fd38cfe43962452e4bfadc6966b2ea0afaf8e0286cb3991c247c8c33ebd/mcp-1.12.2-py3-none-any.whl", hash = "sha256:b86d584bb60193a42bd78aef01882c5c42d614e416cbf0480149839377ab5a5f", size = 158473, upload-time = "2025-07-24T18:29:03.419Z" }, ] [[package]] @@ -589,22 +589,22 @@ wheels = [ [[package]] name = "mypy" -version = "1.16.1" +version = "1.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mypy-extensions" }, { name = "pathspec" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/69/92c7fa98112e4d9eb075a239caa4ef4649ad7d441545ccffbd5e34607cbb/mypy-1.16.1.tar.gz", hash = "sha256:6bd00a0a2094841c5e47e7374bb42b83d64c527a502e3334e1173a0c24437bab", size = 3324747, upload-time = "2025-06-16T16:51:35.145Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e3/034322d5a779685218ed69286c32faa505247f1f096251ef66c8fd203b08/mypy-1.17.0.tar.gz", hash = "sha256:e5d7ccc08ba089c06e2f5629c660388ef1fee708444f1dee0b9203fa031dee03", size = 3352114, upload-time = "2025-07-14T20:34:30.181Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/61/ec1245aa1c325cb7a6c0f8570a2eee3bfc40fa90d19b1267f8e50b5c8645/mypy-1.16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:472e4e4c100062488ec643f6162dd0d5208e33e2f34544e1fc931372e806c0cc", size = 10890557, upload-time = "2025-06-16T16:37:21.421Z" }, - { url = "https://files.pythonhosted.org/packages/6b/bb/6eccc0ba0aa0c7a87df24e73f0ad34170514abd8162eb0c75fd7128171fb/mypy-1.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea16e2a7d2714277e349e24d19a782a663a34ed60864006e8585db08f8ad1782", size = 10012921, upload-time = "2025-06-16T16:51:28.659Z" }, - { url = "https://files.pythonhosted.org/packages/5f/80/b337a12e2006715f99f529e732c5f6a8c143bb58c92bb142d5ab380963a5/mypy-1.16.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08e850ea22adc4d8a4014651575567b0318ede51e8e9fe7a68f25391af699507", size = 11802887, upload-time = "2025-06-16T16:50:53.627Z" }, - { url = "https://files.pythonhosted.org/packages/d9/59/f7af072d09793d581a745a25737c7c0a945760036b16aeb620f658a017af/mypy-1.16.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22d76a63a42619bfb90122889b903519149879ddbf2ba4251834727944c8baca", size = 12531658, upload-time = "2025-06-16T16:33:55.002Z" }, - { url = "https://files.pythonhosted.org/packages/82/c4/607672f2d6c0254b94a646cfc45ad589dd71b04aa1f3d642b840f7cce06c/mypy-1.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2c7ce0662b6b9dc8f4ed86eb7a5d505ee3298c04b40ec13b30e572c0e5ae17c4", size = 12732486, upload-time = "2025-06-16T16:37:03.301Z" }, - { url = "https://files.pythonhosted.org/packages/b6/5e/136555ec1d80df877a707cebf9081bd3a9f397dedc1ab9750518d87489ec/mypy-1.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:211287e98e05352a2e1d4e8759c5490925a7c784ddc84207f4714822f8cf99b6", size = 9479482, upload-time = "2025-06-16T16:47:37.48Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d3/53e684e78e07c1a2bf7105715e5edd09ce951fc3f47cf9ed095ec1b7a037/mypy-1.16.1-py3-none-any.whl", hash = "sha256:5fc2ac4027d0ef28d6ba69a0343737a23c4d1b83672bf38d1fe237bdc0643b37", size = 2265923, upload-time = "2025-06-16T16:48:02.366Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/82efb502b0b0f661c49aa21cfe3e1999ddf64bf5500fc03b5a1536a39d39/mypy-1.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9d4fe5c72fd262d9c2c91c1117d16aac555e05f5beb2bae6a755274c6eec42be", size = 10914150, upload-time = "2025-07-14T20:31:51.985Z" }, + { url = "https://files.pythonhosted.org/packages/03/96/8ef9a6ff8cedadff4400e2254689ca1dc4b420b92c55255b44573de10c54/mypy-1.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d96b196e5c16f41b4f7736840e8455958e832871990c7ba26bf58175e357ed61", size = 10039845, upload-time = "2025-07-14T20:32:30.527Z" }, + { url = "https://files.pythonhosted.org/packages/df/32/7ce359a56be779d38021d07941cfbb099b41411d72d827230a36203dbb81/mypy-1.17.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73a0ff2dd10337ceb521c080d4147755ee302dcde6e1a913babd59473904615f", size = 11837246, upload-time = "2025-07-14T20:32:01.28Z" }, + { url = "https://files.pythonhosted.org/packages/82/16/b775047054de4d8dbd668df9137707e54b07fe18c7923839cd1e524bf756/mypy-1.17.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24cfcc1179c4447854e9e406d3af0f77736d631ec87d31c6281ecd5025df625d", size = 12571106, upload-time = "2025-07-14T20:34:26.942Z" }, + { url = "https://files.pythonhosted.org/packages/a1/cf/fa33eaf29a606102c8d9ffa45a386a04c2203d9ad18bf4eef3e20c43ebc8/mypy-1.17.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3c56f180ff6430e6373db7a1d569317675b0a451caf5fef6ce4ab365f5f2f6c3", size = 12759960, upload-time = "2025-07-14T20:33:42.882Z" }, + { url = "https://files.pythonhosted.org/packages/94/75/3f5a29209f27e739ca57e6350bc6b783a38c7621bdf9cac3ab8a08665801/mypy-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:eafaf8b9252734400f9b77df98b4eee3d2eecab16104680d51341c75702cad70", size = 9503888, upload-time = "2025-07-14T20:32:34.392Z" }, + { url = "https://files.pythonhosted.org/packages/e3/fc/ee058cc4316f219078464555873e99d170bde1d9569abd833300dbeb484a/mypy-1.17.0-py3-none-any.whl", hash = "sha256:15d9d0018237ab058e5de3d8fce61b6fa72cc59cc78fd91f1b474bce12abf496", size = 2283195, upload-time = "2025-07-14T20:31:54.753Z" }, ] [[package]] @@ -913,12 +913,12 @@ wheels = [ [[package]] name = "pywin32" -version = "310" +version = "311" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/b1/68aa2986129fb1011dabbe95f0136f44509afaf072b12b8f815905a39f33/pywin32-310-cp311-cp311-win32.whl", hash = "sha256:1e765f9564e83011a63321bb9d27ec456a0ed90d3732c4b2e312b855365ed8bd", size = 8784284, upload-time = "2025-03-17T00:55:53.124Z" }, - { url = "https://files.pythonhosted.org/packages/b3/bd/d1592635992dd8db5bb8ace0551bc3a769de1ac8850200cfa517e72739fb/pywin32-310-cp311-cp311-win_amd64.whl", hash = "sha256:126298077a9d7c95c53823934f000599f66ec9296b09167810eb24875f32689c", size = 9520748, upload-time = "2025-03-17T00:55:55.203Z" }, - { url = "https://files.pythonhosted.org/packages/90/b1/ac8b1ffce6603849eb45a91cf126c0fa5431f186c2e768bf56889c46f51c/pywin32-310-cp311-cp311-win_arm64.whl", hash = "sha256:19ec5fc9b1d51c4350be7bb00760ffce46e6c95eaf2f0b2f1150657b1a43c582", size = 8455941, upload-time = "2025-03-17T00:55:57.048Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/449a6a91e5d6db51420875c54f6aff7c97a86a3b13a0b4f1a5c13b988de3/pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151", size = 8697031, upload-time = "2025-07-14T20:13:13.266Z" }, + { url = "https://files.pythonhosted.org/packages/51/8f/9bb81dd5bb77d22243d33c8397f09377056d5c687aa6d4042bea7fbf8364/pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503", size = 9508308, upload-time = "2025-07-14T20:13:15.147Z" }, + { url = "https://files.pythonhosted.org/packages/44/7b/9c2ab54f74a138c491aba1b1cd0795ba61f144c711daea84a88b63dc0f6c/pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2", size = 8703930, upload-time = "2025-07-14T20:13:16.945Z" }, ] [[package]] @@ -992,15 +992,15 @@ wheels = [ [[package]] name = "rich" -version = "14.0.0" +version = "14.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/53/830aa4c3066a8ab0ae9a9955976fb770fe9c6102117c8ec4ab3ea62d89e8/rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725", size = 224078, upload-time = "2025-03-30T14:15:14.23Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload-time = "2025-03-30T14:15:12.283Z" }, + { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, ] [[package]] @@ -1079,27 +1079,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.12.3" +version = "0.12.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/2a/43955b530c49684d3c38fcda18c43caf91e99204c2a065552528e0552d4f/ruff-0.12.3.tar.gz", hash = "sha256:f1b5a4b6668fd7b7ea3697d8d98857390b40c1320a63a178eee6be0899ea2d77", size = 4459341, upload-time = "2025-07-11T13:21:16.086Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/cd/01015eb5034605fd98d829c5839ec2c6b4582b479707f7c1c2af861e8258/ruff-0.12.5.tar.gz", hash = "sha256:b209db6102b66f13625940b7f8c7d0f18e20039bb7f6101fbdac935c9612057e", size = 5170722, upload-time = "2025-07-24T13:26:37.456Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/fd/b44c5115539de0d598d75232a1cc7201430b6891808df111b8b0506aae43/ruff-0.12.3-py3-none-linux_armv6l.whl", hash = "sha256:47552138f7206454eaf0c4fe827e546e9ddac62c2a3d2585ca54d29a890137a2", size = 10430499, upload-time = "2025-07-11T13:20:26.321Z" }, - { url = "https://files.pythonhosted.org/packages/43/c5/9eba4f337970d7f639a37077be067e4ec80a2ad359e4cc6c5b56805cbc66/ruff-0.12.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0a9153b000c6fe169bb307f5bd1b691221c4286c133407b8827c406a55282041", size = 11213413, upload-time = "2025-07-11T13:20:30.017Z" }, - { url = "https://files.pythonhosted.org/packages/e2/2c/fac3016236cf1fe0bdc8e5de4f24c76ce53c6dd9b5f350d902549b7719b2/ruff-0.12.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fa6b24600cf3b750e48ddb6057e901dd5b9aa426e316addb2a1af185a7509882", size = 10586941, upload-time = "2025-07-11T13:20:33.046Z" }, - { url = "https://files.pythonhosted.org/packages/c5/0f/41fec224e9dfa49a139f0b402ad6f5d53696ba1800e0f77b279d55210ca9/ruff-0.12.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2506961bf6ead54887ba3562604d69cb430f59b42133d36976421bc8bd45901", size = 10783001, upload-time = "2025-07-11T13:20:35.534Z" }, - { url = "https://files.pythonhosted.org/packages/0d/ca/dd64a9ce56d9ed6cad109606ac014860b1c217c883e93bf61536400ba107/ruff-0.12.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4faaff1f90cea9d3033cbbcdf1acf5d7fb11d8180758feb31337391691f3df0", size = 10269641, upload-time = "2025-07-11T13:20:38.459Z" }, - { url = "https://files.pythonhosted.org/packages/63/5c/2be545034c6bd5ce5bb740ced3e7014d7916f4c445974be11d2a406d5088/ruff-0.12.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40dced4a79d7c264389de1c59467d5d5cefd79e7e06d1dfa2c75497b5269a5a6", size = 11875059, upload-time = "2025-07-11T13:20:41.517Z" }, - { url = "https://files.pythonhosted.org/packages/8e/d4/a74ef1e801ceb5855e9527dae105eaff136afcb9cc4d2056d44feb0e4792/ruff-0.12.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:0262d50ba2767ed0fe212aa7e62112a1dcbfd46b858c5bf7bbd11f326998bafc", size = 12658890, upload-time = "2025-07-11T13:20:44.442Z" }, - { url = "https://files.pythonhosted.org/packages/13/c8/1057916416de02e6d7c9bcd550868a49b72df94e3cca0aeb77457dcd9644/ruff-0.12.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12371aec33e1a3758597c5c631bae9a5286f3c963bdfb4d17acdd2d395406687", size = 12232008, upload-time = "2025-07-11T13:20:47.374Z" }, - { url = "https://files.pythonhosted.org/packages/f5/59/4f7c130cc25220392051fadfe15f63ed70001487eca21d1796db46cbcc04/ruff-0.12.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:560f13b6baa49785665276c963edc363f8ad4b4fc910a883e2625bdb14a83a9e", size = 11499096, upload-time = "2025-07-11T13:20:50.348Z" }, - { url = "https://files.pythonhosted.org/packages/d4/01/a0ad24a5d2ed6be03a312e30d32d4e3904bfdbc1cdbe63c47be9d0e82c79/ruff-0.12.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023040a3499f6f974ae9091bcdd0385dd9e9eb4942f231c23c57708147b06311", size = 11688307, upload-time = "2025-07-11T13:20:52.945Z" }, - { url = "https://files.pythonhosted.org/packages/93/72/08f9e826085b1f57c9a0226e48acb27643ff19b61516a34c6cab9d6ff3fa/ruff-0.12.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:883d844967bffff5ab28bba1a4d246c1a1b2933f48cb9840f3fdc5111c603b07", size = 10661020, upload-time = "2025-07-11T13:20:55.799Z" }, - { url = "https://files.pythonhosted.org/packages/80/a0/68da1250d12893466c78e54b4a0ff381370a33d848804bb51279367fc688/ruff-0.12.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2120d3aa855ff385e0e562fdee14d564c9675edbe41625c87eeab744a7830d12", size = 10246300, upload-time = "2025-07-11T13:20:58.222Z" }, - { url = "https://files.pythonhosted.org/packages/6a/22/5f0093d556403e04b6fd0984fc0fb32fbb6f6ce116828fd54306a946f444/ruff-0.12.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6b16647cbb470eaf4750d27dddc6ebf7758b918887b56d39e9c22cce2049082b", size = 11263119, upload-time = "2025-07-11T13:21:01.503Z" }, - { url = "https://files.pythonhosted.org/packages/92/c9/f4c0b69bdaffb9968ba40dd5fa7df354ae0c73d01f988601d8fac0c639b1/ruff-0.12.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e1417051edb436230023575b149e8ff843a324557fe0a265863b7602df86722f", size = 11746990, upload-time = "2025-07-11T13:21:04.524Z" }, - { url = "https://files.pythonhosted.org/packages/fe/84/7cc7bd73924ee6be4724be0db5414a4a2ed82d06b30827342315a1be9e9c/ruff-0.12.3-py3-none-win32.whl", hash = "sha256:dfd45e6e926deb6409d0616078a666ebce93e55e07f0fb0228d4b2608b2c248d", size = 10589263, upload-time = "2025-07-11T13:21:07.148Z" }, - { url = "https://files.pythonhosted.org/packages/07/87/c070f5f027bd81f3efee7d14cb4d84067ecf67a3a8efb43aadfc72aa79a6/ruff-0.12.3-py3-none-win_amd64.whl", hash = "sha256:a946cf1e7ba3209bdef039eb97647f1c77f6f540e5845ec9c114d3af8df873e7", size = 11695072, upload-time = "2025-07-11T13:21:11.004Z" }, - { url = "https://files.pythonhosted.org/packages/e0/30/f3eaf6563c637b6e66238ed6535f6775480db973c836336e4122161986fc/ruff-0.12.3-py3-none-win_arm64.whl", hash = "sha256:5f9c7c9c8f84c2d7f27e93674d27136fbf489720251544c4da7fb3d742e011b1", size = 10805855, upload-time = "2025-07-11T13:21:13.547Z" }, + { url = "https://files.pythonhosted.org/packages/d4/de/ad2f68f0798ff15dd8c0bcc2889558970d9a685b3249565a937cd820ad34/ruff-0.12.5-py3-none-linux_armv6l.whl", hash = "sha256:1de2c887e9dec6cb31fcb9948299de5b2db38144e66403b9660c9548a67abd92", size = 11819133, upload-time = "2025-07-24T13:25:56.369Z" }, + { url = "https://files.pythonhosted.org/packages/f8/fc/c6b65cd0e7fbe60f17e7ad619dca796aa49fbca34bb9bea5f8faf1ec2643/ruff-0.12.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d1ab65e7d8152f519e7dea4de892317c9da7a108da1c56b6a3c1d5e7cf4c5e9a", size = 12501114, upload-time = "2025-07-24T13:25:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/c5/de/c6bec1dce5ead9f9e6a946ea15e8d698c35f19edc508289d70a577921b30/ruff-0.12.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:962775ed5b27c7aa3fdc0d8f4d4433deae7659ef99ea20f783d666e77338b8cf", size = 11716873, upload-time = "2025-07-24T13:26:01.496Z" }, + { url = "https://files.pythonhosted.org/packages/a1/16/cf372d2ebe91e4eb5b82a2275c3acfa879e0566a7ac94d331ea37b765ac8/ruff-0.12.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73b4cae449597e7195a49eb1cdca89fd9fbb16140c7579899e87f4c85bf82f73", size = 11958829, upload-time = "2025-07-24T13:26:03.721Z" }, + { url = "https://files.pythonhosted.org/packages/25/bf/cd07e8f6a3a6ec746c62556b4c4b79eeb9b0328b362bb8431b7b8afd3856/ruff-0.12.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b13489c3dc50de5e2d40110c0cce371e00186b880842e245186ca862bf9a1ac", size = 11626619, upload-time = "2025-07-24T13:26:06.118Z" }, + { url = "https://files.pythonhosted.org/packages/d8/c9/c2ccb3b8cbb5661ffda6925f81a13edbb786e623876141b04919d1128370/ruff-0.12.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1504fea81461cf4841778b3ef0a078757602a3b3ea4b008feb1308cb3f23e08", size = 13221894, upload-time = "2025-07-24T13:26:08.292Z" }, + { url = "https://files.pythonhosted.org/packages/6b/58/68a5be2c8e5590ecdad922b2bcd5583af19ba648f7648f95c51c3c1eca81/ruff-0.12.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c7da4129016ae26c32dfcbd5b671fe652b5ab7fc40095d80dcff78175e7eddd4", size = 14163909, upload-time = "2025-07-24T13:26:10.474Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d1/ef6b19622009ba8386fdb792c0743f709cf917b0b2f1400589cbe4739a33/ruff-0.12.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ca972c80f7ebcfd8af75a0f18b17c42d9f1ef203d163669150453f50ca98ab7b", size = 13583652, upload-time = "2025-07-24T13:26:13.381Z" }, + { url = "https://files.pythonhosted.org/packages/62/e3/1c98c566fe6809a0c83751d825a03727f242cdbe0d142c9e292725585521/ruff-0.12.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8dbbf9f25dfb501f4237ae7501d6364b76a01341c6f1b2cd6764fe449124bb2a", size = 12700451, upload-time = "2025-07-24T13:26:15.488Z" }, + { url = "https://files.pythonhosted.org/packages/24/ff/96058f6506aac0fbc0d0fc0d60b0d0bd746240a0594657a2d94ad28033ba/ruff-0.12.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c47dea6ae39421851685141ba9734767f960113d51e83fd7bb9958d5be8763a", size = 12937465, upload-time = "2025-07-24T13:26:17.808Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d3/68bc5e7ab96c94b3589d1789f2dd6dd4b27b263310019529ac9be1e8f31b/ruff-0.12.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5076aa0e61e30f848846f0265c873c249d4b558105b221be1828f9f79903dc5", size = 11771136, upload-time = "2025-07-24T13:26:20.422Z" }, + { url = "https://files.pythonhosted.org/packages/52/75/7356af30a14584981cabfefcf6106dea98cec9a7af4acb5daaf4b114845f/ruff-0.12.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5a4c7830dadd3d8c39b1cc85386e2c1e62344f20766be6f173c22fb5f72f293", size = 11601644, upload-time = "2025-07-24T13:26:22.928Z" }, + { url = "https://files.pythonhosted.org/packages/c2/67/91c71d27205871737cae11025ee2b098f512104e26ffd8656fd93d0ada0a/ruff-0.12.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:46699f73c2b5b137b9dc0fc1a190b43e35b008b398c6066ea1350cce6326adcb", size = 12478068, upload-time = "2025-07-24T13:26:26.134Z" }, + { url = "https://files.pythonhosted.org/packages/34/04/b6b00383cf2f48e8e78e14eb258942fdf2a9bf0287fbf5cdd398b749193a/ruff-0.12.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5a655a0a0d396f0f072faafc18ebd59adde8ca85fb848dc1b0d9f024b9c4d3bb", size = 12991537, upload-time = "2025-07-24T13:26:28.533Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/053d6445dc7544fb6594785056d8ece61daae7214859ada4a152ad56b6e0/ruff-0.12.5-py3-none-win32.whl", hash = "sha256:dfeb2627c459b0b78ca2bbdc38dd11cc9a0a88bf91db982058b26ce41714ffa9", size = 11751575, upload-time = "2025-07-24T13:26:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/bc/0f/ab16e8259493137598b9149734fec2e06fdeda9837e6f634f5c4e35916da/ruff-0.12.5-py3-none-win_amd64.whl", hash = "sha256:ae0d90cf5f49466c954991b9d8b953bd093c32c27608e409ae3564c63c5306a5", size = 12882273, upload-time = "2025-07-24T13:26:32.929Z" }, + { url = "https://files.pythonhosted.org/packages/00/db/c376b0661c24cf770cb8815268190668ec1330eba8374a126ceef8c72d55/ruff-0.12.5-py3-none-win_arm64.whl", hash = "sha256:48cdbfc633de2c5c37d9f090ba3b352d1576b0015bfc3bc98eaf230275b7e805", size = 11951564, upload-time = "2025-07-24T13:26:34.994Z" }, ] [[package]] @@ -1250,14 +1250,14 @@ wheels = [ [[package]] name = "sse-starlette" -version = "2.4.1" +version = "3.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/3e/eae74d8d33e3262bae0a7e023bb43d8bdd27980aa3557333f4632611151f/sse_starlette-2.4.1.tar.gz", hash = "sha256:7c8a800a1ca343e9165fc06bbda45c78e4c6166320707ae30b416c42da070926", size = 18635, upload-time = "2025-07-06T09:41:33.631Z" } +sdist = { url = "https://files.pythonhosted.org/packages/42/6f/22ed6e33f8a9e76ca0a412405f31abb844b779d52c5f96660766edcd737c/sse_starlette-3.0.2.tar.gz", hash = "sha256:ccd60b5765ebb3584d0de2d7a6e4f745672581de4f5005ab31c3a25d10b52b3a", size = 20985, upload-time = "2025-07-27T09:07:44.565Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/f1/6c7eaa8187ba789a6dd6d74430307478d2a91c23a5452ab339b6fbe15a08/sse_starlette-2.4.1-py3-none-any.whl", hash = "sha256:08b77ea898ab1a13a428b2b6f73cfe6d0e607a7b4e15b9bb23e4a37b087fd39a", size = 10824, upload-time = "2025-07-06T09:41:32.321Z" }, + { url = "https://files.pythonhosted.org/packages/ef/10/c78f463b4ef22eef8491f218f692be838282cd65480f6e423d7730dfd1fb/sse_starlette-3.0.2-py3-none-any.whl", hash = "sha256:16b7cbfddbcd4eaca11f7b586f3b8a080f1afe952c15813455b162edea619e5a", size = 11297, upload-time = "2025-07-27T09:07:43.268Z" }, ] [[package]] @@ -1276,15 +1276,15 @@ wheels = [ [[package]] name = "starlette" -version = "0.47.1" +version = "0.47.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0a/69/662169fdb92fb96ec3eaee218cf540a629d629c86d7993d9651226a6789b/starlette-0.47.1.tar.gz", hash = "sha256:aef012dd2b6be325ffa16698f9dc533614fb1cebd593a906b90dc1025529a79b", size = 2583072, upload-time = "2025-06-21T04:03:17.337Z" } +sdist = { url = "https://files.pythonhosted.org/packages/04/57/d062573f391d062710d4088fa1369428c38d51460ab6fedff920efef932e/starlette-0.47.2.tar.gz", hash = "sha256:6ae9aa5db235e4846decc1e7b79c4f346adf41e9777aebeb49dfd09bbd7023d8", size = 2583948, upload-time = "2025-07-20T17:31:58.522Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/95/38ef0cd7fa11eaba6a99b3c4f5ac948d8bc6ff199aabd327a29cc000840c/starlette-0.47.1-py3-none-any.whl", hash = "sha256:5e11c9f5c7c3f24959edbf2dffdc01bba860228acf657129467d8a7468591527", size = 72747, upload-time = "2025-06-21T04:03:15.705Z" }, + { url = "https://files.pythonhosted.org/packages/f7/1f/b876b1f83aef204198a42dc101613fefccb32258e5428b5f9259677864b4/starlette-0.47.2-py3-none-any.whl", hash = "sha256:c5847e96134e5c5371ee9fac6fdf1a67336d5815e09eb2a01fdb57a351ef915b", size = 72984, upload-time = "2025-07-20T17:31:56.738Z" }, ] [[package]] From 770e3658eb35fbbb1657c3ab4a3e23f486a21baa Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 27 Jul 2025 15:17:34 +0200 Subject: [PATCH 002/220] MCP: Pass initial instructions to fastmcp constructor --- src/serena/mcp.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/serena/mcp.py b/src/serena/mcp.py index ff8a11ba..f5f785e0 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -172,7 +172,8 @@ class SerenaMCPFactory: Settings.model_config = SettingsConfigDict(env_prefix="FASTMCP_") mcp_settings: Settings = Settings(lifespan=self.server_lifespan, host=host, port=port) - mcp = FastMCP(**mcp_settings.model_dump()) + instructions = self._get_initial_instructions() + mcp = FastMCP(**mcp_settings.model_dump(), instructions=instructions) return mcp @asynccontextmanager @@ -181,6 +182,10 @@ class SerenaMCPFactory: """Manage server startup and shutdown lifecycle.""" yield None # ensures MyPy understands we yield None + @abstractmethod + def _get_initial_instructions(self) -> str: + pass + class SerenaMCPFactorySingleProcess(SerenaMCPFactory): """ @@ -207,6 +212,11 @@ class SerenaMCPFactorySingleProcess(SerenaMCPFactory): assert self.agent is not None yield from self.agent.get_exposed_tool_instances() + def _get_initial_instructions(self) -> str: + assert self.agent is not None + # we don't use the tool (which at the time of writing calls this method), since the tool may be disabled by the config + return self.agent.create_system_prompt() + @asynccontextmanager async def server_lifespan(self, mcp_server: FastMCP) -> AsyncIterator[None]: self._set_mcp_tools(mcp_server) From f30005fe4bee4854872b2ff10fe563b9a9f743f0 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 27 Jul 2025 15:22:17 +0200 Subject: [PATCH 003/220] MCP: Updated readme on using initial instructions --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 788ac9b6..24abe333 100644 --- a/README.md +++ b/README.md @@ -354,11 +354,14 @@ For example, when using `uvx`, you would run claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena-mcp-server --context ide-assistant --project $(pwd) ``` -ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools. - Once in Claude Code, you can ask to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. - Do this whenever you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. +ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools. + As of version `v1.0.52`, claude code reads the instructions of the MCP server, so this is handled automatically. + If you are using an older version, or if Claude fails to read the instructions, you can ask it explicitly + to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. + You may have to do that whenever you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. -ℹ️ **NEW**: an alternative to the above is adding the instructions as part of the system prompt, then you will not need to run the command above or to remember re-running it after compacting. +ℹ️ If Claude is not automatically picking up on the instructions and running Serena's tools, and you don't want to run the command in each session, + an alternative to the above is adding the instructions as part of the system prompt, then you will not need to run the command above or to remember re-running it after compacting. This can be achieved through starting claude code with `claude --append-system-prompt $(uvx --from git+https://github.com/oraios/serena serena print-system-prompt)`. Note that this is **experimental**, Claude may not understand the instructions correctly in this way, and we haven't thoroughly tested the resulting behavior. Please report any issues you encounter. From 9b9eab68bd169077eb4934ba0fabed36bc38fa1f Mon Sep 17 00:00:00 2001 From: architectureman <111078057+architectureman@users.noreply.github.com> Date: Sun, 27 Jul 2025 23:05:20 +0700 Subject: [PATCH 004/220] Update rust_analyzer.py for macOS x64 intel/amd --- src/solidlsp/language_servers/rust_analyzer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index 0f2f2cf1..5a181fc3 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -52,6 +52,14 @@ class RustAnalyzer(SolidLanguageServer): """ deps = RuntimeDependencyCollection( [ + RuntimeDependency( + id="RustAnalyzer", + description="RustAnalyzer for macOS (x64)", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-apple-darwin.gz", + platform_id="osx-x64", + archive_type="gz", + binary_name="rust_analyzer", + ), RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (arm64)", From c08a9b6e5599f6d020844394e20f5d040dae88e5 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 28 Jul 2025 17:15:27 +0200 Subject: [PATCH 005/220] Support list-based commands for RuntimeDependency installation (preferable on Windows where spaces in paths can cause problems) Apply list-based commands in TypeScriptLanguageServer * for launch command * for runtime dependency installation --- src/solidlsp/language_servers/common.py | 12 ++++++++++-- .../language_servers/typescript_language_server.py | 8 ++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/solidlsp/language_servers/common.py b/src/solidlsp/language_servers/common.py index 4ce8f3b4..8163d5b3 100644 --- a/src/solidlsp/language_servers/common.py +++ b/src/solidlsp/language_servers/common.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import os +import platform import subprocess from collections.abc import Sequence from dataclasses import dataclass @@ -21,7 +22,7 @@ class RuntimeDependency: url: str | None = None archive_type: str | None = None binary_name: str | None = None - command: str | None = None + command: str | list[str] | None = None package_name: str | None = None package_version: str | None = None extract_path: str | None = None @@ -71,7 +72,7 @@ class RuntimeDependencyCollection: return results @staticmethod - def _run_command(command: str, cwd: str) -> None: + def _run_command(command: str | list[str], cwd: str) -> None: kwargs = {} if PlatformUtils.get_platform_id().is_windows(): kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW # type: ignore @@ -80,6 +81,13 @@ class RuntimeDependencyCollection: kwargs["user"] = pwd.getpwuid(os.getuid()).pw_name log.info("Running command '%s' in '%s'", command, cwd) + + is_windows = platform.system() == "Windows" + if not isinstance(command, str) and not is_windows: + # Since we are using the shell, we need to convert the command list to a single string + # on Linux/macOS + command = " ".join(command) + completed_process = subprocess.run( command, shell=True, diff --git a/src/solidlsp/language_servers/typescript_language_server.py b/src/solidlsp/language_servers/typescript_language_server.py index 7b700deb..ccb5e8c4 100644 --- a/src/solidlsp/language_servers/typescript_language_server.py +++ b/src/solidlsp/language_servers/typescript_language_server.py @@ -73,7 +73,7 @@ class TypeScriptLanguageServer(SolidLanguageServer): @classmethod def _setup_runtime_dependencies( cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings - ) -> str: + ) -> list[str]: """ Setup runtime dependencies for TypeScript Language Server and return the command to start the server. """ @@ -95,13 +95,13 @@ class TypeScriptLanguageServer(SolidLanguageServer): RuntimeDependency( id="typescript", description="typescript package", - command="npm install --prefix ./ typescript@5.5.4", + command=["npm", "install", "--prefix", "./", "typescript@5.5.4"], platform_id="any", ), RuntimeDependency( id="typescript-language-server", description="typescript-language-server package", - command="npm install --prefix ./ typescript-language-server@4.3.3", + command=["npm", "install", "--prefix", "./", "typescript-language-server@4.3.3"], platform_id="any", ), ] @@ -131,7 +131,7 @@ class TypeScriptLanguageServer(SolidLanguageServer): raise FileNotFoundError( f"typescript-language-server executable not found at {tsserver_executable_path}, something went wrong with the installation." ) - return f"{tsserver_executable_path} --stdio" + return [tsserver_executable_path, "--stdio"] @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: From 68fcc690609c235100b3c094f16490e6c9aacbc3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 28 Jul 2025 18:01:53 +0200 Subject: [PATCH 006/220] Improve logging --- src/solidlsp/language_servers/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/language_servers/common.py b/src/solidlsp/language_servers/common.py index 8163d5b3..1587d0a9 100644 --- a/src/solidlsp/language_servers/common.py +++ b/src/solidlsp/language_servers/common.py @@ -80,7 +80,6 @@ class RuntimeDependencyCollection: import pwd kwargs["user"] = pwd.getpwuid(os.getuid()).pw_name - log.info("Running command '%s' in '%s'", command, cwd) is_windows = platform.system() == "Windows" if not isinstance(command, str) and not is_windows: @@ -88,6 +87,8 @@ class RuntimeDependencyCollection: # on Linux/macOS command = " ".join(command) + log.info("Running command %s in '%s'", f"'{command}'" if isinstance(command, str) else command, cwd) + completed_process = subprocess.run( command, shell=True, From dcbcd5c6d035fc2999c0f074fbfc283ca0a3e698 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 00:12:10 +0200 Subject: [PATCH 007/220] Do not truncate messages when tracing ls communication --- src/solidlsp/ls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index e0a09f4d..ded2645a 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -281,7 +281,7 @@ class SolidLanguageServer(ABC): if config.trace_lsp_communication: def logging_fn(source: str, target: str, msg: StringDict | str): - self.logger.log(f"LSP: {source} -> {target}: {str(msg)[:90]}...", self.logger.logger.level) + self.logger.log(f"LSP: {source} -> {target}: {msg!s}", self.logger.logger.level) else: logging_fn = None From 1324f1a3f2be89e8886277c47569421426bf876f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 00:12:49 +0200 Subject: [PATCH 008/220] Log all messages written to the LS's stderr stream at ERROR level --- src/solidlsp/ls_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 4565d2c5..91574c1c 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -365,7 +365,7 @@ class SolidLanguageServerHandler: line = self.process.stderr.readline() if not line: continue - self._log("LSP stderr: " + line.decode(ENCODING, errors="replace")) + log.error(line.decode(ENCODING, errors="replace")) except (BrokenPipeError, ConnectionResetError): pass log.info("Language server stderr reader thread has terminated") From 72bfb611c273e6fce7d8a69f0647fb1e6e43bfdd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 00:35:29 +0200 Subject: [PATCH 009/220] Fix termination detection in stderr/stdout LS reader threads --- src/solidlsp/ls_handler.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 91574c1c..6a994ed3 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -134,7 +134,7 @@ class SolidLanguageServerHandler: self.notify = LspNotification(self.send_notification) self.process_launch_info = process_launch_info - self.process = None + self.process: subprocess.Popen | None = None self._is_shutting_down = False self.request_id = 1 @@ -326,7 +326,9 @@ class SolidLanguageServerHandler: """ exception: Exception | None = None try: - while self.process and self.process.stdout and self.process.stdout.readable(): + while self.process and self.process.stdout: + if self.process.poll() is not None: # process has terminated + break line = self.process.stdout.readline() if not line: continue @@ -361,14 +363,20 @@ class SolidLanguageServerHandler: Continuously read from the language server process stderr and log the messages """ try: - while self.process and self.process.stderr and self.process.stderr.readable(): + while self.process and self.process.stderr: + if self.process.poll() is not None: + # process has terminated + break line = self.process.stderr.readline() if not line: continue log.error(line.decode(ENCODING, errors="replace")) - except (BrokenPipeError, ConnectionResetError): - pass - log.info("Language server stderr reader thread has terminated") + except Exception as e: + log.error("Error while reading stderr from language server process: %s", e, exc_info=e) + if not self._is_shutting_down: + log.error("Language server stderr reader thread terminated unexpectedly") + else: + log.info("Language server stderr reader thread has terminated") def _handle_body(self, body: bytes) -> None: """ From 53b5bcb71bd1773f0a34f79cf31e22e95f3777e5 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 00:54:48 +0200 Subject: [PATCH 010/220] Update sensai-utils to 1.5.0 --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e79567c2..e6ee41f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "python-dotenv>=1.0.0, <2", "mcp>=1.5.0", "flask>=3.0.0", - "sensai-utils>=1.4.0", + "sensai-utils>=1.5.0", "pydantic>=2.10.6", "types-pyyaml>=6.0.12.20241230", "pyyaml>=6.0.2", diff --git a/uv.lock b/uv.lock index 52fe9264..569fa4a8 100644 --- a/uv.lock +++ b/uv.lock @@ -1104,14 +1104,14 @@ wheels = [ [[package]] name = "sensai-utils" -version = "1.4.0" +version = "1.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/b1/9dba08e2d8ea739b58d1ac393c066169f1ca6506c86f0322d6e56e600ae2/sensai_utils-1.4.0.tar.gz", hash = "sha256:2d32bdcc91fd1428c5cae0181e98623142d2d5f7e115e23d585a842dd9dc59ba", size = 56317, upload-time = "2025-01-27T22:47:51.121Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/dd/faa2e2de71a03af3def212c70777e794dd54ad5ab87927bb5c29f85f24fc/sensai_utils-1.5.0.tar.gz", hash = "sha256:2ca709a0d5807caf1632d665a455c173987b25276ce61693021672e875f0f17b", size = 61677, upload-time = "2025-07-28T22:46:10.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/df/49cc942b653427df0bbd3b9a2fb1e85652df3d4c1d9f6dcbfce1b6155716/sensai_utils-1.4.0-py3-none-any.whl", hash = "sha256:ed6fc57552620e43b33cf364ea0bc0fd7df39391069dd7b621b113ef55547507", size = 63214, upload-time = "2025-01-27T22:47:49.606Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d5/62a0e693230bace8e9a767d6d187a4d9421a7c6ee4b48551f8ff7bd1629a/sensai_utils-1.5.0-py3-none-any.whl", hash = "sha256:c23c51d23d353e7a9b77c70aa8ba7f2d0a8fe4e67ee5bc1ef41c69dba5e4befb", size = 68836, upload-time = "2025-07-28T22:46:08.929Z" }, ] [[package]] @@ -1189,7 +1189,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.3,<3" }, { name = "ruamel-yaml", specifier = ">=0.18.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.0.285" }, - { name = "sensai-utils", specifier = ">=1.4.0" }, + { name = "sensai-utils", specifier = ">=1.5.0" }, { name = "sqlalchemy", marker = "extra == 'agno'", specifier = ">=2.0.40" }, { name = "syrupy", marker = "extra == 'dev'", specifier = ">=4.9.1" }, { name = "tiktoken", specifier = ">=0.9.0" }, From 1a07352034cc9fe43faf71f182832d2a5adb671e Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 00:59:16 +0200 Subject: [PATCH 011/220] Fix broken shell configuration imposed by Claude Code (sets COMSPEC to Git-Bash on Windows) --- src/serena/agent.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/serena/agent.py b/src/serena/agent.py index fe6a9091..66394d62 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -181,6 +181,8 @@ class SerenaAgent: log.info("Available projects: {}".format(", ".join(self.serena_config.project_names))) log.info(f"Loaded tools ({len(self._all_tools)}): {', '.join([tool.get_name_from_cls() for tool in self._all_tools.values()])}") + self._check_shell_settings() + # determine the base toolset defining the set of exposed tools (which e.g. the MCP shall see), # limited by the Serena config, the context (which is fixed for the session) and JetBrains mode tool_inclusion_definitions: list[ToolInclusionDefinition] = [self.serena_config, self._context] @@ -225,6 +227,16 @@ class SerenaAgent: except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + def _check_shell_settings(self) -> None: + # On Windows, Claude Code sets COMSPEC to Git-Bash (often even with a path containing spaces), + # which causes all sorts of trouble, preventing language servers from being launched correctly. + # So we make sure that COMSPEC is unset if it has been set to bash specifically. + if platform.system() == "Windows": + comspec = os.environ.get("COMSPEC", "") + if "bash" in comspec: + os.environ["COMSPEC"] = "" # force use of default shell + log.info("Adjusting COMSPEC environment variable to use the default shell instead of '%s'", comspec) + def _ide_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]: """ In the IDE assistant context, the agent is assumed to work on a single project, and we thus From 39e67acc63bd87c93f36d16f3099408c43dbcf3b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 01:05:51 +0200 Subject: [PATCH 012/220] Update change log --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c1b190..dfbf52c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,15 @@ Status of the `main` branch. Changes prior to the next official version change will appear here. +* Language servers: + * Reliably detect language server termination and propagate the respective error all the way + back to the tool application, where an unexpected termination is handled by restarting the language server + and subsequently retrying the tool application. + Fixes: * Ignore `.git` in check for ignored paths and improve performance of `find_all_non_ignored_files` +* Fix language server startup issues on Windows when using Claude Code (which was due to + default shell reconfiguration imposed by Claude Code) # 0.1.3 From e37c2af571cededebc8547f5db3024d3b4ddf616 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 29 Jul 2025 01:51:44 +0200 Subject: [PATCH 013/220] Update deprecation message --- src/serena/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 7591e0ca..deb3b6e6 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -435,7 +435,7 @@ class ProjectCommands(AutoRegisteringGroup): @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False) @click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING") def index_deprecated(project: str, log_level: str = "WARNING") -> None: - click.echo("Deprecated! Use `project index` instead.") + click.echo("Deprecated! Use `serena project index` instead.") ProjectCommands._index_project(project, log_level) @staticmethod From 3e49c83d89e29c08fbd6710a3ce4df3e342f3ffd Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Tue, 29 Jul 2025 15:55:07 +0200 Subject: [PATCH 014/220] Made initial instructions tool optional --- src/serena/tools/workflow_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 6172cc33..38cc19be 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -5,7 +5,7 @@ Tools supporting the general workflow of the agent import json import platform -from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional class CheckOnboardingPerformedTool(Tool): @@ -122,7 +122,7 @@ class PrepareForNewConversationTool(Tool): return self.prompt_factory.create_prepare_for_new_conversation() -class InitialInstructionsTool(Tool, ToolMarkerDoesNotRequireActiveProject): +class InitialInstructionsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): """ Gets the initial instructions for the current project. Should only be used in settings where the system prompt cannot be set, From 534567f11e087beed8012d968114f9560b4b8beb Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Tue, 29 Jul 2025 15:57:40 +0200 Subject: [PATCH 015/220] Excluded more tools in ide-assistant mode Namely: - prepare_for_new_conversation - summarize_changes - get_current_config --- src/serena/resources/config/contexts/ide-assistant.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index d40cb860..2a6c9e2c 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -22,4 +22,7 @@ excluded_tools: - delete_lines - replace_lines - insert_at_line - - execute_shell_command \ No newline at end of file + - execute_shell_command + - prepare_for_new_conversation + - summarize_changes + - get_current_config From d624c1c0b9241a549f50e1c202860932dbfe040d Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Wed, 30 Jul 2025 12:07:30 +0200 Subject: [PATCH 016/220] Allow passing a list of dirs as prompts_dir, the tail of the list serving as fallbacks --- src/interprompt/multilang_prompt.py | 96 +++++++++++++++++++++++++---- src/interprompt/prompt_factory.py | 6 +- 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/interprompt/multilang_prompt.py b/src/interprompt/multilang_prompt.py index 55777344..e7322efb 100644 --- a/src/interprompt/multilang_prompt.py +++ b/src/interprompt/multilang_prompt.py @@ -1,7 +1,7 @@ import logging import os from enum import Enum -from typing import Any, Generic, TypeVar +from typing import Any, Generic, Literal, TypeVar import yaml from sensai.util.string import ToStringMixin @@ -92,6 +92,9 @@ class _MultiLangContainer(Generic[T], ToStringMixin): raise KeyError(f"Item for language '{lang_code}' already registered for name '{self.name}'") self._lang2item[lang_code] = item + def has_item(self, lang_code: str = DEFAULT_LANG_CODE) -> bool: + return lang_code in self._lang2item + def get_item(self, lang: str = DEFAULT_LANG_CODE, fallback_mode: LanguageFallbackMode = LanguageFallbackMode.EXCEPTION) -> T: """ Gets the item for the given language. @@ -183,6 +186,9 @@ class MultiLangPromptTemplate(ParameterizedTemplateInterface): prompt_template = self.get_prompt_template(lang_code, fallback_mode) return prompt_template.render(**params) + def has_item(self, lang_code: str = DEFAULT_LANG_CODE) -> bool: + return self._prompts_container.has_item(lang_code) + class MultiLangPromptList(_MultiLangContainer[PromptList]): pass @@ -212,44 +218,96 @@ class MultiLangPromptCollection: The prompt names must be unique (for the same language) within the collection. """ - def __init__(self, prompts_dir: str, fallback_mode: LanguageFallbackMode = LanguageFallbackMode.EXCEPTION) -> None: + def __init__(self, prompts_dir: str | list[str], fallback_mode: LanguageFallbackMode = LanguageFallbackMode.EXCEPTION) -> None: """ - :param prompts_dir: the directory containing the prompt templates and prompt lists + :param prompts_dir: the directory containing the prompt templates and prompt lists. + If a list is provided, will look for prompt templates in the dirs from left to right + (first one containing the desired template wins). :param fallback_mode: the fallback mode to use when a prompt template or prompt list is not found for the requested language. May be reset after initialization. """ self._multi_lang_prompt_templates: dict[str, MultiLangPromptTemplate] = {} self._multi_lang_prompt_lists: dict[str, MultiLangPromptList] = {} - self._load_from_disc(prompts_dir) + if isinstance(prompts_dir, str): + prompts_dir = [prompts_dir] + + # Add prompts from multiple directories, prioritizing names from the left. + # If name collisions appear in the first directory, an error is raised (so the first directory should have no + # internal collisions, this helps in avoiding errors) + # For all following directories, on a collision the new value will be ignored. + # This also means that for the following directories, there is no error check on collisions internal to them. + # We assume that they are correct (i.e., they have no internal collisions). + first_prompts_dir, fallback_prompt_dirs = prompts_dir[0], prompts_dir[1:] + self._load_from_disc(first_prompts_dir, on_name_collision="raise") + for fallback_prompt_dir in fallback_prompt_dirs: + # already loaded prompts have priority + self._load_from_disc(fallback_prompt_dir, on_name_collision="skip") + self.fallback_mode = fallback_mode - def _add_prompt_template(self, name: str, template_str: str, lang_code: str = DEFAULT_LANG_CODE) -> None: + def _add_prompt_template( + self, + name: str, + template_str: str, + lang_code: str = DEFAULT_LANG_CODE, + on_name_collision: Literal["skip", "overwrite", "raise"] = "raise", + ) -> None: """ :param name: name of the prompt template :param template_str: the Jinja template string :param lang_code: the language code for which to add the prompt template. + :param on_name_collision: how to deal with name/lang_code collisions """ + allow_overwrite = False prompt_template = PromptTemplate(name, template_str) mlpt = self._multi_lang_prompt_templates.get(name) if mlpt is None: mlpt = MultiLangPromptTemplate(name) self._multi_lang_prompt_templates[name] = mlpt - mlpt.add_prompt_template(prompt_template, lang_code=lang_code) + if mlpt.has_item(lang_code): + if on_name_collision == "raise": + raise KeyError(f"Prompt '{name}' for {lang_code} already exists!") + if on_name_collision == "skip": + log.debug(f"Skipping prompt '{name}' since it already exists.") + return + elif on_name_collision == "overwrite": + allow_overwrite = True + mlpt.add_prompt_template(prompt_template, lang_code=lang_code, allow_overwrite=allow_overwrite) - def _add_prompt_list(self, name: str, prompt_list: list[str], lang_code: str = DEFAULT_LANG_CODE) -> None: + def _add_prompt_list( + self, + name: str, + prompt_list: list[str], + lang_code: str = DEFAULT_LANG_CODE, + on_name_collision: Literal["skip", "overwrite", "raise"] = "raise", + ) -> None: """ :param name: name of the prompt list :param prompt_list: a list of prompts :param lang_code: the language code for which to add the prompt list. + :param on_name_collision: how to deal with name/lang_code collisions """ + allow_overwrite = False multilang_prompt_list = self._multi_lang_prompt_lists.get(name) if multilang_prompt_list is None: multilang_prompt_list = MultiLangPromptList(name) self._multi_lang_prompt_lists[name] = multilang_prompt_list - multilang_prompt_list.add_item(PromptList(prompt_list), lang_code=lang_code) + if multilang_prompt_list.has_item(lang_code): + if on_name_collision == "raise": + raise KeyError(f"Prompt '{name}' for {lang_code} already exists!") + if on_name_collision == "skip": + log.debug(f"Skipping prompt '{name}' since it already exists.") + return + elif on_name_collision == "overwrite": + allow_overwrite = True + multilang_prompt_list.add_item(PromptList(prompt_list), lang_code=lang_code, allow_overwrite=allow_overwrite) - def _load_from_disc(self, prompts_dir: str) -> None: - """Loads all prompt templates and prompt lists from yaml files in the given directory.""" + def _load_from_disc(self, prompts_dir: str, on_name_collision: Literal["skip", "overwrite", "raise"] = "raise") -> None: + """Loads all prompt templates and prompt lists from yaml files in the given directory. + + :param prompts_dir: + :param on_name_collision: how to deal with name/lang_code collisions + """ for fn in os.listdir(prompts_dir): if not fn.endswith((".yml", ".yaml")): log.debug(f"Skipping non-YAML file: {fn}") @@ -266,9 +324,11 @@ class MultiLangPromptCollection: # add the data to the collection for prompt_name, prompt_template_or_list in prompts_data.items(): if isinstance(prompt_template_or_list, list): - self._add_prompt_list(prompt_name, prompt_template_or_list, lang_code=lang_code) + self._add_prompt_list(prompt_name, prompt_template_or_list, lang_code=lang_code, on_name_collision=on_name_collision) elif isinstance(prompt_template_or_list, str): - self._add_prompt_template(prompt_name, prompt_template_or_list, lang_code=lang_code) + self._add_prompt_template( + prompt_name, prompt_template_or_list, lang_code=lang_code, on_name_collision=on_name_collision + ) else: raise ValueError( f"Invalid prompt type for {prompt_name} in file {path} (should be str or list): {prompt_template_or_list}" @@ -306,6 +366,18 @@ class MultiLangPromptCollection: """The PromptList object for the given prompt name and language code.""" return self.get_multilang_prompt_list(prompt_name).get_item(lang_code) + def _has_prompt_list(self, prompt_name: str, lang_code: str = DEFAULT_LANG_CODE) -> bool: + multi_lang_prompt_list = self._multi_lang_prompt_lists.get(prompt_name) + if multi_lang_prompt_list is None: + return False + return multi_lang_prompt_list.has_item(lang_code) + + def _has_prompt_template(self, prompt_name: str, lang_code: str = DEFAULT_LANG_CODE) -> bool: + multi_lang_prompt_template = self._multi_lang_prompt_templates.get(prompt_name) + if multi_lang_prompt_template is None: + return False + return multi_lang_prompt_template.has_item(lang_code) + def render_prompt_template( self, prompt_name: str, diff --git a/src/interprompt/prompt_factory.py b/src/interprompt/prompt_factory.py index 3aa9da08..1769e7f2 100644 --- a/src/interprompt/prompt_factory.py +++ b/src/interprompt/prompt_factory.py @@ -10,9 +10,11 @@ log = logging.getLogger(__name__) class PromptFactoryBase: """Base class for auto-generated prompt factory classes.""" - def __init__(self, prompts_dir: str, lang_code: str = DEFAULT_LANG_CODE, fallback_mode=LanguageFallbackMode.EXCEPTION): + def __init__(self, prompts_dir: str | list[str], lang_code: str = DEFAULT_LANG_CODE, fallback_mode=LanguageFallbackMode.EXCEPTION): """ - :param prompts_dir: the directory containing the prompt templates and prompt lists + :param prompts_dir: the directory containing the prompt templates and prompt lists. + If a list is provided, will look for prompt templates in the dirs from left to right + (first one containing the desired template wins). :param lang_code: the language code to use for retrieving the prompt templates and prompt lists. Leave as `default` for single-language use cases. :param fallback_mode: the fallback mode to use when a prompt template or prompt list is not found for the requested language. From caa2c545e4c8870421e47c87e1b698f4c6fbc5d3 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Wed, 30 Jul 2025 12:10:15 +0200 Subject: [PATCH 017/220] New feature: allow users to override prompts. Extended CLI for that --- scripts/gen_prompt_factory.py | 4 +- src/serena/cli.py | 91 ++++++++++++++++++++++++++++++++++- src/serena/constants.py | 3 +- src/serena/prompt_factory.py | 7 ++- 4 files changed, 99 insertions(+), 6 deletions(-) diff --git a/scripts/gen_prompt_factory.py b/scripts/gen_prompt_factory.py index 4b1354fd..4877d4d1 100644 --- a/scripts/gen_prompt_factory.py +++ b/scripts/gen_prompt_factory.py @@ -7,14 +7,14 @@ from pathlib import Path from sensai.util import logging from interprompt import autogenerate_prompt_factory_module -from serena.constants import PROMPT_TEMPLATES_DIR, REPO_ROOT +from serena.constants import PROMPT_TEMPLATES_DIR_INTERNAL, REPO_ROOT log = logging.getLogger(__name__) def main(): autogenerate_prompt_factory_module( - prompts_dir=PROMPT_TEMPLATES_DIR, + prompts_dir=PROMPT_TEMPLATES_DIR_INTERNAL, target_module_path=str(Path(REPO_ROOT) / "src" / "serena" / "generated" / "generated_prompt_factory.py"), ) diff --git a/src/serena/cli.py b/src/serena/cli.py index deb3b6e6..6b46a8e8 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1,3 +1,4 @@ +import glob import os import shutil import subprocess @@ -16,6 +17,8 @@ from serena.config.serena_config import ProjectConfig, SerenaConfig, SerenaPaths from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, + PROMPT_TEMPLATES_DIR_IN_USER_HOME, + PROMPT_TEMPLATES_DIR_INTERNAL, SERENA_LOG_FORMAT, SERENA_MANAGED_DIR_IN_HOME, SERENAS_OWN_CONTEXT_YAMLS_DIR, @@ -476,12 +479,98 @@ class ToolCommands(AutoRegisteringGroup): ToolRegistry().print_tool_overview() +class PromptCommands(AutoRegisteringGroup): + def __init__(self) -> None: + super().__init__(name="prompts", help="Commands related to Serena's prompts that are outside of contexts and modes.") + + @staticmethod + def _get_user_prompt_yaml_path(prompt_yaml_name: str) -> str: + os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + return os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, prompt_yaml_name) + + @staticmethod + @click.command("list", help="Lists yamls that are used for defining prompts.") + def list() -> None: + serena_prompt_yaml_names = [os.path.basename(f) for f in glob.glob(PROMPT_TEMPLATES_DIR_INTERNAL + "/*.yml")] + for prompt_yaml_name in serena_prompt_yaml_names: + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if os.path.exists(user_prompt_yaml_path): + click.echo(f"{user_prompt_yaml_path} merged with default prompts in {prompt_yaml_name}") + else: + click.echo(prompt_yaml_name) + + @staticmethod + @click.command("create-override", help="Create an override of an internal prompts yaml for customizing Serena's prompts") + @click.argument("prompt_yaml_name") + def create_override(prompt_yaml_name: str) -> None: + """ + :param prompt_yaml_name: The yaml name of the prompt you want to override. Call the `list` command for discovering valid prompt yaml names. + :return: + """ + # for convenience, we can pass names without .yml + if not prompt_yaml_name.endswith(".yml"): + prompt_yaml_name = prompt_yaml_name + ".yml" + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if os.path.exists(user_prompt_yaml_path): + raise FileExistsError(f"{user_prompt_yaml_path} already exists.") + serena_prompt_yaml_path = os.path.join(PROMPT_TEMPLATES_DIR_INTERNAL, prompt_yaml_name) + shutil.copyfile(serena_prompt_yaml_path, user_prompt_yaml_path) + _open_in_editor(user_prompt_yaml_path) + + @staticmethod + @click.command("edit-override", help="Edit an existing prompt override file") + @click.argument("prompt_yaml_name") + def edit_override(prompt_yaml_name: str) -> None: + """ + :param prompt_yaml_name: The yaml name of the prompt override to edit. + :return: + """ + # for convenience, we can pass names without .yml + if not prompt_yaml_name.endswith(".yml"): + prompt_yaml_name = prompt_yaml_name + ".yml" + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if not os.path.exists(user_prompt_yaml_path): + click.echo(f"Override file '{prompt_yaml_name}' not found. Create it with: prompts create-override {prompt_yaml_name}") + return + _open_in_editor(user_prompt_yaml_path) + + @staticmethod + @click.command("list-overrides", help="List existing prompt override files") + def list_overrides() -> None: + os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + serena_prompt_yaml_names = [os.path.basename(f) for f in glob.glob(PROMPT_TEMPLATES_DIR_INTERNAL + "/*.yml")] + override_files = glob.glob(os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, "*.yml")) + for file_path in override_files: + if os.path.basename(file_path) in serena_prompt_yaml_names: + click.echo(file_path) + + @staticmethod + @click.command("delete-override", help="Delete a prompt override file") + @click.argument("prompt_yaml_name") + def delete_override(prompt_yaml_name: str) -> None: + """ + + :param prompt_yaml_name: The yaml name of the prompt override to delete." + :return: + """ + # for convenience, we can pass names without .yml + if not prompt_yaml_name.endswith(".yml"): + prompt_yaml_name = prompt_yaml_name + ".yml" + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if not os.path.exists(user_prompt_yaml_path): + click.echo(f"Override file '{prompt_yaml_name}' not found.") + return + os.remove(user_prompt_yaml_path) + click.echo(f"Deleted override file '{prompt_yaml_name}'.") + + # Expose groups so we can reference them in pyproject.toml mode = ModeCommands() context = ContextCommands() project = ProjectCommands() config = SerenaConfigCommands() tools = ToolCommands() +prompts = PromptCommands() # Expose toplevel commands for the same reason top_level = TopLevelCommands() @@ -489,7 +578,7 @@ start_mcp_server = top_level.start_mcp_server index_project = project.index_deprecated # needed for the help script to work - register all subcommands to the top-level group -for subgroup in (mode, context, project, config, tools): +for subgroup in (mode, context, project, config, tools, prompts): top_level.add_command(subgroup) diff --git a/src/serena/constants.py b/src/serena/constants.py index 9b0dd43f..d25f5c45 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -10,7 +10,8 @@ SERENA_MANAGED_DIR_IN_HOME = str(_serena_in_home_managed_dir) # TODO: Path-related constants should be moved to SerenaPaths; don't add further constants here. REPO_ROOT = str(_repo_root_path) -PROMPT_TEMPLATES_DIR = str(_serena_pkg_path / "resources" / "config" / "prompt_templates") +PROMPT_TEMPLATES_DIR_INTERNAL = str(_serena_pkg_path / "resources" / "config" / "prompt_templates") +PROMPT_TEMPLATES_DIR_IN_USER_HOME = str(_serena_in_home_managed_dir / "prompt_templates") SERENAS_OWN_CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts") """The contexts that are shipped with the Serena package, i.e. the default contexts.""" USER_CONTEXT_YAMLS_DIR = str(_serena_in_home_managed_dir / "contexts") diff --git a/src/serena/prompt_factory.py b/src/serena/prompt_factory.py index f4f02ca7..40f51e7a 100644 --- a/src/serena/prompt_factory.py +++ b/src/serena/prompt_factory.py @@ -1,4 +1,6 @@ -from serena.constants import PROMPT_TEMPLATES_DIR +import os + +from serena.constants import PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL from serena.generated.generated_prompt_factory import PromptFactory @@ -8,4 +10,5 @@ class SerenaPromptFactory(PromptFactory): """ def __init__(self) -> None: - super().__init__(PROMPT_TEMPLATES_DIR) + os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + super().__init__(prompts_dir=[PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL]) From d26dc3536af36d90c2156630326ce73ee89cbe96 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 31 Jul 2025 11:52:51 +0200 Subject: [PATCH 018/220] Readme: more info on CLI, initial instructions, and included_optional_tools --- README.md | 109 ++++++++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 24abe333..cfe3aa44 100644 --- a/README.md +++ b/README.md @@ -202,38 +202,27 @@ This and other settings can be adjusted in the [configuration](#configuration) a git clone https://github.com/oraios/serena cd serena ``` -2. Optionally create the configuration file in your home directory, i.e. - - * `~/.serena/serena_config.yml` on Linux and macOS, or - * `%USERPROFILE%\.serena\serena_config.yml` on Windows. - - by copying the template and then adjusting it according to your needs: +2. Optionally edit the configuration file in your home directory with ```shell - mkdir ~/.serena - cp src/serena/resources/serena_config.template.yml ~/.serena/serena_config.yml + uv run serena config edit ``` If you just want the default config, you can skip this part, and a config file will be created when you first run Serena. 3. Run the server with `uv`: ```shell - uv run serena-mcp-server + uv run serena start-mcp-server ``` - When running from outside the serena installation directory, be sure to pass it, i.e. use + When running from outside the serena installation directory, be sure to pass it, i.e., use ```shell - uv run --directory /abs/path/to/serena serena-mcp-server + uv run --directory /abs/path/to/serena serena start-mcp-server ``` ##### Using uvx `uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation. -* Windows: - ```shell - uvx --from git+https://github.com/oraios/serena serena-mcp-server.exe - ``` -* Other operating systems: - ```shell - uvx --from git+https://github.com/oraios/serena serena-mcp-server - ``` +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server +``` ##### Using Docker (Experimental) @@ -243,7 +232,7 @@ You can run the Serena MCP server directly via docker as follows, assuming that the projects you want to work on are all located in `/path/to/your/projects`: ```shell -docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena-mcp-server --transport stdio +docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena start-mcp-server --transport stdio ``` Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides: @@ -263,12 +252,12 @@ therefore needs to be configured with a launch command. When using instead the SSE mode, which uses HTTP-based communication, you control the server lifecycle yourself, i.e. you start the server and provide the client with the URL to connect to it. -Simply provide `serena-mcp-server` with the `--transport sse` option and optionally provide the port. +Simply provide `start-mcp-server` with the `--transport sse` option and optionally provide the port. For example, to run the Serena MCP server in SSE mode on port 9121 using a local installation, you would run this command from the Serena directory, ```shell -uv run serena-mcp-server --transport sse --port 9121 +uv run serena start-mcp-server --transport sse --port 9121 ``` and then configure your client to connect to `http://localhost:9121/sse`. @@ -289,12 +278,22 @@ Serena's behavior (active tools and prompts as well as logging configuration, et 1. The `serena_config.yml` for general settings that apply to all clients and projects. It is located in your user directory under `.serena/serena_config.yml`. If you do not explicitly create the file, it will be auto-generated when you first run Serena. -2. In the arguments passed to the `serena-mcp-server` in your client's config (see below), + You can edit it directly or use + ```shell + uvx --from git+https://github.com/oraios/serena serena config edit + ``` + (or use the `--directory` command version). +2. In the arguments passed to the `start-mcp-server` in your client's config (see below), which will apply to all sessions started by the respective client. In particular, the [context](#contexts) parameter should be set appropriately for Serena to be best adjusted to existing tools and capabilities of your client. See for a detailed explanation. You can override all entries from the `serena_config.yml` through command line arguments. 3. In the `.serena/project.yml` file within your project. This will hold project-level configuration that is used whenever - that project is activated. + that project is activated. This file will be autogenerated when you first use Serena on that project, but you can also + generate it explicitly with + ```shell + uvx --from git+https://github.com/oraios/serena serena project generate-yml + ``` + (or use the `--directory` command version). 4. Through the currently active set of [modes](#modes). @@ -323,20 +322,17 @@ project, the file `.serena/project.yml` will be generated. You can adjust the la same name. If you are mostly working with the same project, you can also configure to always activate a project at startup -by passing `--project ` to the `serena-mcp-server` command in your client's MCP config. +by passing `--project ` to the `start-mcp-server` command in your client's MCP config. ℹ️ For larger projects, we recommend that you index your project to accelerate Serena's tools; otherwise the first tool application may be very slow. -To do so, run one of these commands the project directory or pass the path to the project as an argument: +To do so, run this from the project directory (or pass the path to the project as an argument): -* When using a local installation: - ```shell - uv run --directory /abs/path/to/serena index-project - ``` -* When using uvx: - ```shell - uvx --from git+https://github.com/oraios/serena index-project - ``` +```shell +uvx --from git+https://github.com/oraios/serena serena project index +``` + +(or use the `--directory` command version). ### Claude Code @@ -351,18 +347,16 @@ claude mcp add serena -- --context ide-assistant --project $ where `` is your way of [running the Serena MCP server](#running-the-serena-mcp-server). For example, when using `uvx`, you would run ```shell -claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena-mcp-server --context ide-assistant --project $(pwd) +claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena start-mcp-server --context ide-assistant --project $(pwd) ``` ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools. - As of version `v1.0.52`, claude code reads the instructions of the MCP server, so this is handled automatically. + As of version `v1.0.52`, claude code reads the instructions of the MCP server, so this **is handled automatically**. If you are using an older version, or if Claude fails to read the instructions, you can ask it explicitly - to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. - You may have to do that whenever you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. - -ℹ️ If Claude is not automatically picking up on the instructions and running Serena's tools, and you don't want to run the command in each session, - an alternative to the above is adding the instructions as part of the system prompt, then you will not need to run the command above or to remember re-running it after compacting. - This can be achieved through starting claude code with `claude --append-system-prompt $(uvx --from git+https://github.com/oraios/serena serena print-system-prompt)`. Note that this is **experimental**, Claude may not understand the instructions correctly in this way, and we haven't thoroughly tested the resulting behavior. Please report any issues you encounter. + to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. + If you want to make use of that, you will have to enable the corresponding tool explicitly by adding `initial_instructions` to the `included_optional_tools` + in your config. + Note that you may have to make Claude read the instructions you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. ### Claude Desktop @@ -377,7 +371,7 @@ Add the `serena` MCP server configuration, using a [run command](#running-the-se "mcpServers": { "serena": { "command": "/abs/path/to/uv", - "args": ["run", "--directory", "/abs/path/to/serena", "serena-mcp-server"] + "args": ["run", "--directory", "/abs/path/to/serena", "serena", "start-mcp-server"] } } } @@ -388,7 +382,7 @@ Add the `serena` MCP server configuration, using a [run command](#running-the-se "mcpServers": { "serena": { "command": "/abs/path/to/uvx", - "args": ["--from", "git+https://github.com/oraios/serena", "serena-mcp-server"] + "args": ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server"] } } } @@ -399,7 +393,7 @@ Add the `serena` MCP server configuration, using a [run command](#running-the-se "mcpServers": { "serena": { "command": "docker", - "args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena-mcp-server", "--transport", "stdio"] + "args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena", "start-mcp-server", "--transport", "stdio"] } } } @@ -581,14 +575,20 @@ When launching Serena, specify modes using `--mode `; multiple modes #### Customization You can create your own contexts and modes to precisely tailor Serena to your needs in two ways: -* **Adding to Serena's configuration directory**: Create new `.yml` files in the `config/contexts/` or `config/modes/` directories within your local Serena repository. These custom contexts/modes will be automatically registered and available for use by their name (filename without the `.yml` extension). They will also appear in listings of available contexts/modes. -* **Using external YAML files**: When starting Serena, you can provide an absolute path to a custom `.yml` file for a context or mode. +* You can use Serena's CLI to manage modes and contexts. Check out + ```shell + uvx --from git+https://github.com/oraios/serena serena mode --help + uvx --from git+https://github.com/oraios/serena serena context --help + ``` + *NOTE*: Custom contexts/modes are simply YAML files in `/.serena`, they are automatically registered and available for use by their name (filename without the `.yml` extension). If you don't want to use Serena's CLI, you can create and manage them in any way you see fit. +* **Using external YAML files**: When starting Serena, you can also provide an absolute path to a custom `.yml` file for a context or mode. A context or mode YAML file typically defines: * `name`: (Optional if filename is used) The name of the context/mode. * `prompt`: A string that will be incorporated into Serena's system prompt. -* `description`: (Optional) A brief description. +* `description`: (Optional) A brief description, not passed to the LLM. * `excluded_tools`: A list of tool names (strings) to disable when this context/mode is active. +* `included_optional_tools`: A list of tool names that are disabled by default and have to be explicitly enabled by the user. This customization allows for deep integration and adaptation of Serena to specific project requirements or personal preferences. @@ -741,11 +741,10 @@ subscriptions. ### Subscription-Based Coding Agents -The most prominent subscription-based coding agents are parts of IDEs like +Many prominent subscription-based coding agents are parts of IDEs like Windsurf, Cursor and VSCode. Serena's functionality is similar to Cursor's Agent, Windsurf's Cascade or -VSCode's -upcoming [agent mode](https://code.visualstudio.com/blogs/2025/02/24/introducing-copilot-agent-mode). +VSCode's agent mode. Serena has the advantage of not requiring a subscription. A potential disadvantage is that it @@ -753,7 +752,7 @@ is not directly integrated into an IDE, so the inspection of newly written code is not as seamless. More technical differences are: -* Serena is not bound to a specific IDE. +* Serena is not bound to a specific IDE or CLI. Serena's MCP server can be used with any MCP client (including some IDEs), and the Agno-based agent provides additional ways of applying its functionality. * Serena is not bound to a specific large language model or API. @@ -829,7 +828,7 @@ For details on contributing, see [here](/CONTRIBUTING.md). ## Full List of Tools -Here is the full list of Serena's tools with a short description (output of `uv run serena-list-tools`): +Here is the full list of Serena's tools with a short description (output of `uv run serena tools list`): * `activate_project`: Activates a project by name. * `check_onboarding_performed`: Checks whether project onboarding was already performed. @@ -843,9 +842,7 @@ Here is the full list of Serena's tools with a short description (output of `uv * `get_active_project`: Gets the name of the currently active project (if any) and lists existing projects * `get_current_config`: Prints the current configuration of the agent, including the active modes, tools, and context. * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file or directory. - * `initial_instructions`: Gets the initial instructions for the current project. - Should only be used in settings where the system prompt cannot be set, - e.g. in clients you have no control over, like Claude Desktop. + * `initial_instructions`: Gets the initial instructions for the current project. Disabled by default, has to be enabled explicitly by using `included_optional_tools` * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. * `insert_at_line`: Inserts content at a given line in a file. * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. From c69e30a7bc50338bd4a3af936a3f788969e1b573 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 31 Jul 2025 11:57:11 +0200 Subject: [PATCH 019/220] Readme: moved uvx install option above the local installation option [ci skip] --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index cfe3aa44..1b6777cb 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,16 @@ Note that no matter how you run the MCP server, Serena will, by default, start a MCP server (since many clients fail to clean up processes correctly). This and other settings can be adjusted in the [configuration](#configuration) and/or by providing [command-line arguments](#command-line-arguments). +##### Using uvx + +`uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation. + +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server +``` + +Explore the CLI to see some of the customization options that serena provides (more info on them below). + ###### Local Installation 1. Clone the repository and change into it. @@ -214,15 +224,7 @@ This and other settings can be adjusted in the [configuration](#configuration) a When running from outside the serena installation directory, be sure to pass it, i.e., use ```shell uv run --directory /abs/path/to/serena serena start-mcp-server - ``` - -##### Using uvx - -`uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation. - -```shell -uvx --from git+https://github.com/oraios/serena serena start-mcp-server -``` + ``` ##### Using Docker (Experimental) From 9cf5ee22a6ca8e9d2b742cc6bf5e56ab8b6b37a2 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 31 Jul 2025 23:10:32 +0200 Subject: [PATCH 020/220] Hotfix: pin mcp dependency to 1.12.2 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e6ee41f3..1c2dfced 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "pyright>=1.1.396,<2", "overrides>=7.7.0,<8", "python-dotenv>=1.0.0, <2", - "mcp>=1.5.0", + "mcp==1.12.2", "flask>=3.0.0", "sensai-utils>=1.5.0", "pydantic>=2.10.6", diff --git a/uv.lock b/uv.lock index b4ab8659..1f278a04 100644 --- a/uv.lock +++ b/uv.lock @@ -1174,7 +1174,7 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "jinja2", marker = "extra == 'dev'" }, { name = "joblib", specifier = ">=1.5.1" }, - { name = "mcp", specifier = ">=1.5.0" }, + { name = "mcp", specifier = "==1.12.2" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.16.1" }, { name = "overrides", specifier = ">=7.7.0,<8" }, { name = "pathspec", specifier = ">=0.12.1" }, From 9af709eb7b8d7221c3dfd782593fb609dec55202 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 31 Jul 2025 23:35:08 +0200 Subject: [PATCH 021/220] MCP: update to 1.12.3, pass kwargs without dumping settings The code also works for 1.12.2, but the new version has typed params in the constructor, and is thus better. --- pyproject.toml | 2 +- src/serena/mcp.py | 4 +--- uv.lock | 8 ++++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c2dfced..8872384c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "pyright>=1.1.396,<2", "overrides>=7.7.0,<8", "python-dotenv>=1.0.0, <2", - "mcp==1.12.2", + "mcp==1.12.3", "flask>=3.0.0", "sensai-utils>=1.5.0", "pydantic>=2.10.6", diff --git a/src/serena/mcp.py b/src/serena/mcp.py index f5f785e0..9926af8e 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -170,10 +170,8 @@ class SerenaMCPFactory: # `.env` files (e.g. containing LOG_LEVEL) that are not supposed to override the MCP settings; # retain only FASTMCP_ prefix for already set environment variables. Settings.model_config = SettingsConfigDict(env_prefix="FASTMCP_") - - mcp_settings: Settings = Settings(lifespan=self.server_lifespan, host=host, port=port) instructions = self._get_initial_instructions() - mcp = FastMCP(**mcp_settings.model_dump(), instructions=instructions) + mcp = FastMCP(lifespan=self.server_lifespan, host=host, port=port, instructions=instructions) return mcp @asynccontextmanager diff --git a/uv.lock b/uv.lock index 1f278a04..a1169097 100644 --- a/uv.lock +++ b/uv.lock @@ -558,7 +558,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.12.2" +version = "1.12.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -573,9 +573,9 @@ dependencies = [ { name = "starlette" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/66/85/f36d538b1286b7758f35c1b69d93f2719d2df90c01bd074eadd35f6afc35/mcp-1.12.2.tar.gz", hash = "sha256:a4b7c742c50ce6ed6d6a6c096cca0e3893f5aecc89a59ed06d47c4e6ba41edcc", size = 426202, upload-time = "2025-07-24T18:29:05.175Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/19/9955e2df5384ff5dd25d38f8e88aaf89d2d3d9d39f27e7383eaf0b293836/mcp-1.12.3.tar.gz", hash = "sha256:ab2e05f5e5c13e1dc90a4a9ef23ac500a6121362a564447855ef0ab643a99fed", size = 427203, upload-time = "2025-07-31T18:36:36.795Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/cf/3fd38cfe43962452e4bfadc6966b2ea0afaf8e0286cb3991c247c8c33ebd/mcp-1.12.2-py3-none-any.whl", hash = "sha256:b86d584bb60193a42bd78aef01882c5c42d614e416cbf0480149839377ab5a5f", size = 158473, upload-time = "2025-07-24T18:29:03.419Z" }, + { url = "https://files.pythonhosted.org/packages/8f/8b/0be74e3308a486f1d127f3f6767de5f9f76454c9b4183210c61cc50999b6/mcp-1.12.3-py3-none-any.whl", hash = "sha256:5483345bf39033b858920a5b6348a303acacf45b23936972160ff152107b850e", size = 158810, upload-time = "2025-07-31T18:36:34.915Z" }, ] [[package]] @@ -1174,7 +1174,7 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "jinja2", marker = "extra == 'dev'" }, { name = "joblib", specifier = ">=1.5.1" }, - { name = "mcp", specifier = "==1.12.2" }, + { name = "mcp", specifier = "==1.12.3" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.16.1" }, { name = "overrides", specifier = ">=7.7.0,<8" }, { name = "pathspec", specifier = ">=0.12.1" }, From 88ceb4920031c1b059c783e6292f442699be1c63 Mon Sep 17 00:00:00 2001 From: Jason Stillwell Date: Sat, 2 Aug 2025 02:30:34 -0500 Subject: [PATCH 022/220] Don't subvert the gem system and try to run the binary directly. Instead use the framework provided by ruby gems to execute the provide binary. --- src/solidlsp/language_servers/solargraph.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index 56a1b7f7..7b79c324 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -87,19 +87,7 @@ class Solargraph(SolidLanguageServer): logger.log("Installing Solargraph...", logging.INFO) subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) - # Get the gem executable path directly - result = subprocess.run(["gem", "which", "solargraph"], check=True, capture_output=True, text=True, cwd=repository_root_path) - gem_path = result.stdout.strip() - bin_dir = os.path.join(os.path.dirname(os.path.dirname(gem_path)), "bin") - executable_path = os.path.join(bin_dir, "solargraph") - - if not os.path.exists(executable_path): - raise RuntimeError(f"Solargraph executable not found at {executable_path}") - - # Ensure the executable has the right permissions - os.chmod(executable_path, os.stat(executable_path).st_mode | stat.S_IEXEC) - - return executable_path + return "gem exec solargraph" except subprocess.CalledProcessError as e: raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e From c44a6be58f4a4447a9d057b3ca98331edb2297c5 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 17:38:20 +0200 Subject: [PATCH 023/220] Docs on connecting Serena to chatgpt --- docs/serena_on_chatgpt.md | 101 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/serena_on_chatgpt.md diff --git a/docs/serena_on_chatgpt.md b/docs/serena_on_chatgpt.md new file mode 100644 index 00000000..64f7e248 --- /dev/null +++ b/docs/serena_on_chatgpt.md @@ -0,0 +1,101 @@ + +# Connecting Serena MCP Server to ChatGPT via MCPO & Cloudflare Tunnel + +This guide explains how to expose a **locally running Serena MCP server** (powered by MCPO) to the internet using **Cloudflare Tunnel**, and how to connect it to **ChatGPT as a Custom GPT with tool access**. + +Once configured, ChatGPT becomes a powerful **coding agent** with direct access to your codebase, shell, and file system — so **read the security notes carefully**. + +--- +## Prerequisites + +Make sure you have [uv](https://docs.astral.sh/uv/getting-started/installation/) +and [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/) installed. + +## 1. Start the Serena MCP Server via MCPO + +Run the following command to launch Serena as http server (assuming port 8000): + +```bash +uvx mcpo --port 8000 --api-key -- \ + uvx --from git+https://github.com/oraios/serena \ + serena start-mcp-server --project $(pwd) +``` + +- `--api-key` is required to secure the server. +- `--project` should point to the root of your codebase. + +You can also use other options, and you don't have to pass `--project` if you want to work on multiple projects +or want to activate it later. See + +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server --help +``` + +--- + +## 2. Expose the Server Using Cloudflare Tunnel + +Run: + +```bash +cloudflared tunnel --url http://localhost:8000 +``` + +This will give you a **public HTTPS URL** like: + +``` +https://serena-agent-tunnel.trycloudflare.com +``` + +Your server is now securely exposed to the internet. + +--- + +## 3. Connect It to ChatGPT (Custom GPT) + +### Steps: + +1. Go to [ChatGPT → Explore GPTs → Create](https://chat.openai.com/gpts/editor) +2. During setup, click **“Add APIs”** +3. Set up **API Key authentication** with the auth type as **Bearer** and enter the api key you used to start the MCPO server. +4. In the **Schema** section, click on **import from URL** and paste `/openapi.json` with the URL you got from the previous step. +5. Add the following line to the top of the imported JSON schema: + ``` + "servers": ["url": ""], + ``` + +ChatGPT will read the schema and create functions automatically. + +--- + +## Security Warning — Read Carefully + +Depending on your configuration and enabled tools, Serena's MCP server may: +- Execute **arbitrary shell commands** +- Read, write, and modify **files in your codebase** + +This gives ChatGPT the same powers as a remote developer on your machine. + +### ⚠️ Key Rules: +- **NEVER expose your API key** +- **Only expose this server when needed**, and monitor its use. + +In your project’s `.serena/project.yml` or global config, you can disable tools like: + +```yaml +excluded_tools: + - execute_shell_command + - ... +read_only: true +``` + +This is strongly recommended if you want a read-only or safer agent. + + +--- + +## Final Thoughts + +With this setup, ChatGPT becomes a coding assistant **running on your local code** — able to index, search, edit, and even run shell commands depending on your configuration. + +Use responsibly, and keep security in mind. From e41d09d9372d580e92835ca4980c00a432a9b03d Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 18:55:34 +0200 Subject: [PATCH 024/220] Formatting --- src/solidlsp/language_servers/solargraph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index 7b79c324..a4f7e694 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,7 +7,6 @@ import json import logging import os import pathlib -import stat import subprocess import threading From 46967fa3cbe5af325bcc9c07722ee9ee7d42aff9 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 18:56:09 +0200 Subject: [PATCH 025/220] Allow overriding tool descriptions in the context, cli command for tool description --- src/serena/agent.py | 10 ++++++++ src/serena/cli.py | 24 ++++++++++++++++++- src/serena/config/context_mode.py | 12 ++++++++-- src/serena/mcp.py | 11 +++++++-- .../resources/config/contexts/agent.yml | 4 +++- .../config/contexts/context.template.yml | 6 ++++- .../resources/config/contexts/desktop-app.yml | 3 ++- .../config/contexts/ide-assistant.yml | 3 +++ 8 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 66394d62..7e900f34 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -227,6 +227,12 @@ class SerenaAgent: except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + def get_context(self) -> SerenaAgentContext: + return self._context + + def get_tool_description_override(self, tool_name: str) -> str | None: + return self._context.tool_description_overrides.get(tool_name, None) + def _check_shell_settings(self) -> None: # On Windows, Claude Code sets COMSPEC to Git-Bash (often even with a path containing spaces), # which causes all sorts of trouble, preventing language servers from being launched correctly. @@ -583,3 +589,7 @@ class SerenaAgent: if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() + + def get_tool_by_name(self, tool_name: str) -> Tool: + tool_class = ToolRegistry().get_tool_class_by_name(tool_name) + return self.get_tool(tool_class) diff --git a/src/serena/cli.py b/src/serena/cli.py index 6b46a8e8..457b3acb 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -26,7 +26,7 @@ from serena.constants import ( USER_CONTEXT_YAMLS_DIR, USER_MODE_YAMLS_DIR, ) -from serena.mcp import SerenaMCPFactorySingleProcess +from serena.mcp import SerenaMCPFactory, SerenaMCPFactorySingleProcess from serena.project import Project from serena.tools import ToolRegistry from serena.util.logging import MemoryLogHandler @@ -478,6 +478,28 @@ class ToolCommands(AutoRegisteringGroup): else: ToolRegistry().print_tool_overview() + @staticmethod + @click.command( + "description", + help="Print the description of a tool, optionally with a specific context (the latter may modify the default description).", + ) + @click.argument("tool_name", type=str) + @click.option("--context", type=str, default=None, help="Context name or path to context file.") + def description(tool_name: str, context: str | None = None) -> None: + # Load the context + serena_context = None + if context: + serena_context = SerenaAgentContext.load(context) + + agent = SerenaAgent( + project=None, + serena_config=SerenaConfig(web_dashboard=False, log_level=logging.INFO), + context=serena_context, + ) + tool = agent.get_tool_by_name(tool_name) + mcp_tool = SerenaMCPFactory.make_mcp_tool(tool) + click.echo(mcp_tool.description) + class PromptCommands(AutoRegisteringGroup): def __init__(self) -> None: diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index de84f6a7..30caba0b 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -4,7 +4,7 @@ Context and Mode configuration loader import os from copy import copy -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self @@ -134,6 +134,8 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str description: str = "" + tool_description_overrides: dict[str, str] = field(default_factory=dict) + """Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.""" def _tostring_includes(self) -> list[str]: return ["name"] @@ -147,6 +149,9 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): def from_json_dict(cls, data: dict) -> Self: data = copy(data) data["excluded_tools"] = set(data["excluded_tools"]) + # Ensure backwards compatibility for tool_description_overrides + if "tool_description_overrides" not in data: + data["tool_description_overrides"] = {} return cls(**data) @classmethod @@ -155,7 +160,10 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): with open(yaml_path, encoding="utf-8") as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) - return cls(name=name, **data) + prompt = data["prompt"] + description = data.get("description", "") + tool_description_overrides = data.get("tool_description_overrides", {}) + return cls(name=name, prompt=prompt, description=description, tool_description_overrides=tool_description_overrides) @classmethod def get_path(cls, name: str) -> str: diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 9926af8e..6c75556f 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -68,10 +68,16 @@ class SerenaMCPFactory: # Mount the tool description as a combination of the docstring description and # the return value description, if it exists. - if docstring.description: - func_doc = f"{docstring.description.strip().strip('.')}." + agent = cast(SerenaAgent, tool.agent) + overridden_description = agent.get_context().tool_description_overrides.get(func_name, None) + + if overridden_description is not None: + func_doc = overridden_description + elif docstring.description: + func_doc = docstring.description else: func_doc = "" + func_doc = func_doc.strip().strip(".") if docstring.returns and (docstring_returns_descr := docstring.returns.description): # Only add a space before "Returns" if func_doc is not empty prefix = " " if func_doc else "" @@ -113,6 +119,7 @@ class SerenaMCPFactory: for tool in self._iter_tools(): mcp_tool = self.make_mcp_tool(tool) mcp._tool_manager._tools[tool.get_name()] = mcp_tool + log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}") @abstractmethod def _instantiate_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> None: diff --git a/src/serena/resources/config/contexts/agent.yml b/src/serena/resources/config/contexts/agent.yml index d3da9da9..e2984283 100644 --- a/src/serena/resources/config/contexts/agent.yml +++ b/src/serena/resources/config/contexts/agent.yml @@ -3,4 +3,6 @@ prompt: | You are running in agent context where the system prompt is provided externally. You should use symbolic tools when possible for code understanding and modification. excluded_tools: - - initial_instructions \ No newline at end of file + - initial_instructions + +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index 52e94535..f9dee588 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -1,4 +1,8 @@ # See Serena's documentation for more details on concept of contexts. description: Description of the context, not used in the code. prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] + +# mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). +# Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index ea1567fb..8105ca8d 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -10,4 +10,5 @@ prompt: | If complex relationships that the user asked for should be visualized or explained, consider creating a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] +tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 2a6c9e2c..14d65130 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -26,3 +26,6 @@ excluded_tools: - prepare_for_new_conversation - summarize_changes - get_current_config + +tool_description_overrides: {} + From e9e05c2f3fca4a72b10900359c2644f138b412bc Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 2 Aug 2025 20:32:34 +0200 Subject: [PATCH 026/220] Allow system prompt to depend on available tools and tool categories (as indicated by ToolMarkers) * Introduce additional tool markers that represent relevant categories * Apply some conditions to current system prompt --- src/serena/agent.py | 18 ++++++++++-- .../generated/generated_prompt_factory.py | 4 ++- .../config/prompt_templates/system_prompt.yml | 29 ++++++++++++------- src/serena/tools/jetbrains_tools.py | 8 ++--- src/serena/tools/symbol_tools.py | 20 ++++++++----- src/serena/tools/tools_base.py | 24 +++++++++++++-- src/solidlsp/language_servers/solargraph.py | 1 - 7 files changed, 76 insertions(+), 28 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 66394d62..5f27a1d1 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -25,7 +25,8 @@ from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, T from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory -from serena.tools import ActivateProjectTool, Tool, ToolRegistry +from serena.tools import ActivateProjectTool, Tool, ToolMarker, ToolRegistry +from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler from solidlsp import SolidLanguageServer @@ -336,10 +337,23 @@ class SerenaAgent: return list(self._modes) def create_system_prompt(self) -> str: - return self.prompt_factory.create_system_prompt( + available_tools = [tool.get_name_from_cls() for tool in self.get_exposed_tool_instances()] + + # determine available tool markers (i.e. tool categories) + available_markers = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in self.get_exposed_tool_instances(): + if isinstance(tool, marker_class): + available_markers.add(marker_class.__name__) + + log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) + system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._context.prompt, mode_system_prompts=[mode.prompt for mode in self._modes], + available_tools=available_tools, + available_markers=available_markers, ) + log.info("System prompt:\n%s", system_prompt) def _update_active_tools(self) -> None: """ diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 6bb7a76b..3a5e1442 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -32,5 +32,7 @@ class PromptFactory(PromptFactoryBase): def create_prepare_for_new_conversation(self) -> str: return self._render_prompt("prepare_for_new_conversation", locals()) - def create_system_prompt(self, *, context_system_prompt: Any, mode_system_prompts: Any) -> str: + def create_system_prompt( + self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 8f3dd504..e912f7ca 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,32 +5,38 @@ prompts: system_prompt: | You are a professional coding agent concerned with one particular codebase. You have access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory - files containing general information about the codebase. You operate in a frugal and intelligent manner, always + files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate content that is not needed for the task at hand. When reading code in order to answer a user question or task, you should try reading only the necessary code. Some tasks may require you to understand the architecture of large parts of the codebase, while for others, it may be enough to read a small set of symbols or a single file. Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on - intelligent step-by-step acquisition of information. However, if you already read a file, it does not make + intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), - as you already have the information. + as you already have the information.{% endif %} - I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! CONSIDER INSTEAD USING THE OVERVIEW TOOL AND - SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! + I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! + {% if 'ToolMarkerSymbolicRead' in available_markers %} + CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS! THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES! + {% endif %} You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and the relations between them, and then only reading the bodies of symbols that are necessary to answer the question - or complete the task. You can also use the standard tools like list_dir, find_file and search_for_pattern if you need to. + or complete the task. + You can use the standard tools like list_dir, find_file and search_for_pattern if you need to. When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory. For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions. - If you are unsure about a symbol's name or location (to the extent that substring_matching for the symbol name is not enough), you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase. This way you can first find candidates for symbols or files, - and then proceed with the symbolic tools. + {% if 'search_for_pattern' in available_tools %} + If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast + and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files, + and then proceed with the symbolic tools.{% endif %} + {% endif %} - Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbols` tool for more details + {% if 'ToolMarkerSymbolicRead' in available_markers %} + Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -40,10 +46,13 @@ prompts: you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding to read the desired methods with `include_body=True` You can understand relationships between symbols by using the `find_referencing_symbols` tool. + {% endif %} + {% if 'read_memory' in available_tools %} You generally have access to memories and it may be useful for you to read them, but also only if they help you to answer the question or complete the task. You can infer which memories are relevant to the current task by reading the memory names and descriptions. + {% endif %} The context and modes of operation are described below. From them you can infer how to interact with your user and which tasks and kinds of interactions are expected of you. diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 319aa590..8b9f2d1a 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,10 +1,10 @@ import json -from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional +from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient -class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): +class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -70,7 +70,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): +class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds symbols that reference the given symbol """ @@ -100,7 +100,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerOptional): +class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Retrieves an overview of the top-level symbols within a specified file """ diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 4d2a5ffc..45e04f3e 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -8,7 +8,13 @@ from collections.abc import Sequence from copy import copy from typing import Any -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerCanEdit +from serena.tools import ( + SUCCESS_RESULT, + TOOL_DEFAULT_MAX_ANSWER_LENGTH, + Tool, + ToolMarkerSymbolicEdit, + ToolMarkerSymbolicRead, +) from solidlsp.ls_types import SymbolKind @@ -42,7 +48,7 @@ class RestartLanguageServerTool(Tool): return SUCCESS_RESULT -class GetSymbolsOverviewTool(Tool): +class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): """ Gets an overview of the top-level symbols defined in a given file or directory. """ @@ -70,7 +76,7 @@ class GetSymbolsOverviewTool(Tool): return self._limit_length(result_json_str, max_answer_chars) -class FindSymbolTool(Tool): +class FindSymbolTool(Tool, ToolMarkerSymbolicRead): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -150,7 +156,7 @@ class FindSymbolTool(Tool): return self._limit_length(result, max_answer_chars) -class FindReferencingSymbolsTool(Tool): +class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): """ Finds symbols that reference the symbol at the given location (optionally filtered by type). """ @@ -204,7 +210,7 @@ class FindReferencingSymbolsTool(Tool): return self._limit_length(result, max_answer_chars) -class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): +class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit): """ Replaces the full definition of a symbol. """ @@ -232,7 +238,7 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): +class InsertAfterSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content after the end of the definition of a given symbol. """ @@ -257,7 +263,7 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): +class InsertBeforeSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content before the beginning of the definition of a given symbol. """ diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index cd025970..1f55e604 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -73,22 +73,40 @@ class Component(ABC): TOOL_DEFAULT_MAX_ANSWER_LENGTH = int(2e5) -class ToolMarkerCanEdit: +class ToolMarker: + """ + Base class for tool markers. + """ + + +class ToolMarkerCanEdit(ToolMarker): """ Marker class for all tools that can perform editing operations on files. """ -class ToolMarkerDoesNotRequireActiveProject: +class ToolMarkerDoesNotRequireActiveProject(ToolMarker): pass -class ToolMarkerOptional: +class ToolMarkerOptional(ToolMarker): """ Marker class for optional tools that are disabled by default. """ +class ToolMarkerSymbolicRead(ToolMarker): + """ + Marker class for tools that perform symbol read operations. + """ + + +class ToolMarkerSymbolicEdit(ToolMarkerCanEdit): + """ + Marker class for tools that perform symbolic edit operations. + """ + + class ApplyMethodProtocol(Protocol): """Callable protocol for the apply method of a tool.""" diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index 7b79c324..a4f7e694 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,7 +7,6 @@ import json import logging import os import pathlib -import stat import subprocess import threading From 1bff8fa77422c8566692a4d67050f411ffa7ef67 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 2 Aug 2025 20:42:47 +0200 Subject: [PATCH 027/220] Support use of available_tools and available_markers in context/mode prompts --- src/serena/agent.py | 43 +++++++++++++------ src/serena/config/context_mode.py | 8 ++++ src/serena/resources/config/modes/editing.yml | 8 ++-- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 5f27a1d1..96d769e8 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime +from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode @@ -92,6 +93,23 @@ class MemoriesManager: return f"Memory {name} deleted." +class AvailableTools: + def __init__(self, tools: list[Tool]): + """ + :param tools: the list of available tools + """ + self.tools = tools + self.tool_names = [tool.get_name_from_cls() for tool in tools] + self.tool_marker_names = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in tools: + if isinstance(tool, marker_class): + self.tool_marker_names.add(marker_class.__name__) + + def __len__(self) -> int: + return len(self.tools) + + class SerenaAgent: def __init__( self, @@ -193,7 +211,7 @@ class SerenaAgent: tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains")) self._base_tool_set = ToolSet.default().apply(*tool_inclusion_definitions) - self._exposed_tools = {tc: t for tc, t in self._all_tools.items() if self._base_tool_set.includes_name(t.get_name())} + self._exposed_tools = AvailableTools([t for t in self._all_tools.values() if self._base_tool_set.includes_name(t.get_name())]) log.info(f"Number of exposed tools: {len(self._exposed_tools)}") # create executor for starting the language server and running tools in another thread @@ -302,7 +320,7 @@ class SerenaAgent: If a client should attempt to use a tool that is dynamically disabled (e.g. because a project is activated that disables it), it will receive an error. """ - return list(self._exposed_tools.values()) + return list(self._exposed_tools.tools) def get_active_project(self) -> Project | None: """ @@ -336,24 +354,21 @@ class SerenaAgent: """ return list(self._modes) + def _format_prompt(self, prompt_template: str) -> str: + template = JinjaTemplate(prompt_template) + return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names) + def create_system_prompt(self) -> str: - available_tools = [tool.get_name_from_cls() for tool in self.get_exposed_tool_instances()] - - # determine available tool markers (i.e. tool categories) - available_markers = set() - for marker_class in iter_subclasses(ToolMarker): - for tool in self.get_exposed_tool_instances(): - if isinstance(tool, marker_class): - available_markers.add(marker_class.__name__) - + available_markers = self._exposed_tools.tool_marker_names log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( - context_system_prompt=self._context.prompt, - mode_system_prompts=[mode.prompt for mode in self._modes], - available_tools=available_tools, + context_system_prompt=self._format_prompt(self._context.prompt), + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + available_tools=self._exposed_tools.tool_names, available_markers=available_markers, ) log.info("System prompt:\n%s", system_prompt) + return system_prompt def _update_active_tools(self) -> None: """ diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index de84f6a7..394c609e 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -39,6 +39,10 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: @@ -133,6 +137,10 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index d53750fd..464f4537 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -14,7 +14,7 @@ prompt: | use the regex-based approach that is described below. Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbols` tool for more details + Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -34,7 +34,8 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - Now let us discuss the regex-based approach. + {% if 'replace_regex' in available_tools %} + Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. You use other tools to find the relevant content and @@ -104,7 +105,8 @@ prompt: | Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! -excluded_tools: + {% endif %} +excluded_tools: - replace_lines - insert_at_line - delete_lines From eb0f44cb793e2fb34603d53fb471396ca40e74f5 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 20:03:19 +0200 Subject: [PATCH 028/220] Fixed appending of "." to docstring and mock in test... We should get rid of mocks eventually --- src/serena/mcp.py | 5 +++-- test/serena/test_mcp.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 6c75556f..fb42cbe9 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -68,8 +68,7 @@ class SerenaMCPFactory: # Mount the tool description as a combination of the docstring description and # the return value description, if it exists. - agent = cast(SerenaAgent, tool.agent) - overridden_description = agent.get_context().tool_description_overrides.get(func_name, None) + overridden_description = tool.agent.get_context().tool_description_overrides.get(func_name, None) if overridden_description is not None: func_doc = overridden_description @@ -78,6 +77,8 @@ class SerenaMCPFactory: else: func_doc = "" func_doc = func_doc.strip().strip(".") + if func_doc: + func_doc += "." if docstring.returns and (docstring_returns_descr := docstring.returns.description): # Only add a space before "Returns" if func_doc is not empty prefix = " " if func_doc else "" diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index 37b6eca8..731a1cc6 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -4,16 +4,28 @@ import pytest from mcp.server.fastmcp.tools.base import Tool as MCPTool from serena.agent import Tool, ToolRegistry +from serena.config.context_mode import SerenaAgentContext from serena.mcp import SerenaMCPFactory make_tool = SerenaMCPFactory.make_mcp_tool +# Create a mock agent for tool initialization +class MockAgent: + def __init__(self): + self.project_config = None + self.serena_config = None + + @staticmethod + def get_context() -> SerenaAgentContext: + return SerenaAgentContext.load_default() + + class BaseMockTool(Tool): """A mock Tool class for testing.""" - def __init__(self) -> None: - pass + def __init__(self): + super().__init__(MockAgent()) class BasicTool(BaseMockTool): @@ -274,13 +286,6 @@ def is_test_mock_class(tool_class: type) -> bool: @pytest.mark.parametrize("tool_class", ToolRegistry().get_all_tool_classes()) def test_make_tool_all_tools(tool_class) -> None: """Test that make_tool works for all tools in the codebase.""" - - # Create a mock agent for tool initialization - class MockAgent: - def __init__(self): - self.project_config = None - self.serena_config = None - # Create an instance of the tool tool_instance = tool_class(MockAgent()) From 4410f37ab3b26f97a11903b1faa0e99f01c93742 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 22:49:42 +0200 Subject: [PATCH 029/220] Context: back to using **data in from_yaml, remove to/from json_dict methods Needed since some fields come from parent class, easy to make mistakes (like I did in the past commit) --- src/serena/config/context_mode.py | 33 ++++--------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 30caba0b..e04f3383 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -44,17 +44,6 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - return cls(**data) - def print_overview(self) -> None: """Print an overview of the mode.""" print(f"{self.name}:\n {self.description}") @@ -140,30 +129,16 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - # Ensure backwards compatibility for tool_description_overrides - if "tool_description_overrides" not in data: - data["tool_description_overrides"] = {} - return cls(**data) - @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" with open(yaml_path, encoding="utf-8") as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) - prompt = data["prompt"] - description = data.get("description", "") - tool_description_overrides = data.get("tool_description_overrides", {}) - return cls(name=name, prompt=prompt, description=description, tool_description_overrides=tool_description_overrides) + # Ensure backwards compatibility for tool_description_overrides + if "tool_description_overrides" not in data: + data["tool_description_overrides"] = {} + return cls(name=name, **data) @classmethod def get_path(cls, name: str) -> str: From 99757f35ebd53173c397379d4884c70cacc3981e Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 00:14:11 +0200 Subject: [PATCH 030/220] Made more tools optional and shortened several docstrings --- docs/serena_on_chatgpt.md | 3 +- scripts/demo_run_tools.py | 10 ++--- src/serena/config/context_mode.py | 3 +- .../config/contexts/context.template.yml | 3 ++ .../resources/config/contexts/desktop-app.yml | 3 ++ .../config/contexts/ide-assistant.yml | 5 --- .../resources/config/modes/mode.template.yml | 3 ++ src/serena/tools/cmd_tools.py | 11 +---- src/serena/tools/config_tools.py | 8 ++-- src/serena/tools/file_tools.py | 20 +++------ src/serena/tools/memory_tools.py | 11 +---- src/serena/tools/symbol_tools.py | 44 +++++++++---------- src/serena/tools/workflow_tools.py | 2 +- 13 files changed, 52 insertions(+), 74 deletions(-) diff --git a/docs/serena_on_chatgpt.md b/docs/serena_on_chatgpt.md index 64f7e248..ec872696 100644 --- a/docs/serena_on_chatgpt.md +++ b/docs/serena_on_chatgpt.md @@ -18,7 +18,7 @@ Run the following command to launch Serena as http server (assuming port 8000): ```bash uvx mcpo --port 8000 --api-key -- \ uvx --from git+https://github.com/oraios/serena \ - serena start-mcp-server --project $(pwd) + serena start-mcp-server --context chatgpt --project $(pwd) ``` - `--api-key` is required to secure the server. @@ -63,6 +63,7 @@ Your server is now securely exposed to the internet. ``` "servers": ["url": ""], ``` + **Important**: don't include a trailing slash at the end of the URL! ChatGPT will read the schema and create functions automatically. diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index 394fe553..d52cfb00 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -9,7 +9,7 @@ from pprint import pprint from serena.agent import SerenaAgent from serena.config.serena_config import SerenaConfig from serena.constants import REPO_ROOT -from serena.tools import FindFileTool, FindReferencingSymbolsTool, SearchForPatternTool +from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, SearchForPatternTool if __name__ == "__main__": agent = SerenaAgent(project=REPO_ROOT, serena_config=SerenaConfig(gui_log_window_enabled=False, web_dashboard=False)) @@ -18,13 +18,9 @@ if __name__ == "__main__": find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) find_file_tool = agent.get_tool(FindFileTool) search_pattern_tool = agent.get_tool(SearchForPatternTool) + overview_tool = agent.get_tool(GetSymbolsOverviewTool) result = agent.execute_task( - lambda: search_pattern_tool.apply( - r"def request_full_.*?\).*?\)", - restrict_search_to_code_files=False, - relative_path="src/solidlsp", - paths_include_glob="**/ls.py", - ) + lambda: overview_tool.apply("src/solidlsp/ls.py"), ) pprint(json.loads(result)) diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index e04f3383..fbe34091 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -3,8 +3,7 @@ Context and Mode configuration loader """ import os -from copy import copy -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index f9dee588..c01a1177 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -3,6 +3,9 @@ description: Description of the context, not used in the code. prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context. excluded_tools: [] +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] + # mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). # Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index 8105ca8d..c3adcfb4 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -11,4 +11,7 @@ prompt: | a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. excluded_tools: [] +included_optional_tools: + - switch_modes + tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 14d65130..27a13673 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -19,13 +19,8 @@ prompt: | excluded_tools: - create_text_file - read_file - - delete_lines - - replace_lines - - insert_at_line - execute_shell_command - prepare_for_new_conversation - - summarize_changes - - get_current_config tool_description_overrides: {} diff --git a/src/serena/resources/config/modes/mode.template.yml b/src/serena/resources/config/modes/mode.template.yml index 431afdeb..6d836ca3 100644 --- a/src/serena/resources/config/modes/mode.template.yml +++ b/src/serena/resources/config/modes/mode.template.yml @@ -2,3 +2,6 @@ description: Description of the mode, not used in the code. prompt: Prompt that will form part of the message sent to the model when activating this mode. excluded_tools: [] + +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] \ No newline at end of file diff --git a/src/serena/tools/cmd_tools.py b/src/serena/tools/cmd_tools.py index 0db0da08..214d54e2 100644 --- a/src/serena/tools/cmd_tools.py +++ b/src/serena/tools/cmd_tools.py @@ -19,15 +19,8 @@ class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit): max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Execute a shell command and return its output. - - IMPORTANT: you should always consider the memory about suggested shell commands before using this tool. - If this memory was not loaded in the current conversation, you should load it using the `read_memory` tool - before using this tool. - - You should have at least once looked at the suggested shell commands from the corresponding memory - created during the onboarding process before using this tool. - Never execute unsafe shell commands like `rm -rf /` or similar! Generally be very careful with deletions. + Execute a shell command and return its output. If there is a memory about suggested commands, read that first. + Never execute unsafe shell commands like `rm -rf /` or similar! :param command: the shell command to execute :param cwd: the working directory to execute the command in. If None, the project root will be used. diff --git a/src/serena/tools/config_tools.py b/src/serena/tools/config_tools.py index 51b782ad..8a3ddeb7 100644 --- a/src/serena/tools/config_tools.py +++ b/src/serena/tools/config_tools.py @@ -1,7 +1,7 @@ import json from serena.config.context_mode import SerenaAgentMode -from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): @@ -35,7 +35,7 @@ class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return result_str -class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): +class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): """ Removes a project from the Serena configuration. """ @@ -50,7 +50,7 @@ class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return f"Successfully removed project '{project_name}' from configuration." -class SwitchModesTool(Tool): +class SwitchModesTool(Tool, ToolMarkerOptional): """ Activates modes by providing a list of their names """ @@ -71,7 +71,7 @@ class SwitchModesTool(Tool): return result_str -class GetCurrentConfigTool(Tool): +class GetCurrentConfigTool(Tool, ToolMarkerOptional): """ Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. """ diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index f79ffe06..b5770ab5 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -14,7 +14,7 @@ from fnmatch import fnmatch from pathlib import Path from serena.text_utils import search_files -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, EditedFileContext, Tool, ToolMarkerCanEdit +from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional from serena.util.file_system import scan_directory @@ -29,7 +29,6 @@ class ReadFileTool(Tool): """ Reads the given file or a chunk of it. Generally, symbolic operations like find_symbol or find_referencing_symbols should be preferred if you know which symbols you are looking for. - Reading the entire file is only recommended if there is no other way to get the content required for the task. :param relative_path: the relative path to the file to read :param start_line: the 0-based index of the first line to be retrieved. @@ -60,14 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): def apply(self, relative_path: str, content: str) -> str: """ - Write a new file (or overwrite an existing file). For existing files, it is strongly recommended - to use symbolic operations like replace_symbol_body or insert_after_symbol/insert_before_symbol, if possible. - You can also use insert_at_line to insert content at a specific line for existing files if the symbolic operations - are not the right choice for what you want to do. - - If ever used on an existing file, the content has to be the complete content of that file (so it - may never end with something like "The remaining content of the file is left unchanged."). - For operations that just replace a part of a file, use the replace_lines or the symbolic editing tools instead. + Write a new file or overwrite an existing file. :param relative_path: the relative path to the file to create :param content: the (utf-8-encoded) content to write to the file @@ -200,7 +192,7 @@ class ReplaceRegexTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class DeleteLinesTool(Tool, ToolMarkerCanEdit): +class DeleteLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Deletes a range of lines within a file. """ @@ -228,7 +220,7 @@ class DeleteLinesTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class ReplaceLinesTool(Tool, ToolMarkerCanEdit): +class ReplaceLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Replaces a range of lines within a file with new content. """ @@ -259,7 +251,7 @@ class ReplaceLinesTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAtLineTool(Tool, ToolMarkerCanEdit): +class InsertAtLineTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Inserts content at a given line in a file. """ @@ -351,7 +343,7 @@ class SearchForPatternTool(Tool): For example, for finding classes or methods from a name pattern. Setting to False is a better choice if you also want to search in non-code files, like in html or yaml files, which is why it is the default. - :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested). + :return: A mapping of file paths to lists of matched consecutive lines. """ abs_path = os.path.join(self.get_project_root(), relative_path) if not os.path.exists(abs_path): diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index 63bba4fe..3cf611f0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -10,15 +10,8 @@ class WriteMemoryTool(Tool): def apply(self, memory_name: str, content: str, max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH) -> str: """ - Write some information about this project that can be useful for future tasks to a memory. - Use markdown formatting for the content. - The information should be short and to the point. - The memory name should be meaningful, such that from the name you can infer what the information is about. - It is better to have multiple small memories than to have a single large one because - memories will be read one by one and we only ever want to read relevant memories. - - This tool is either called during the onboarding process or when you have identified - something worth remembering about the project from the past conversation. + Write some information about this project that can be useful for future tasks to a memory in md format. + The memory name should be meaningful. """ if len(content) > max_answer_chars: raise ValueError( diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 4d2a5ffc..0dfb41a3 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -4,6 +4,7 @@ Language server-related tools import dataclasses import json +import os from collections.abc import Sequence from copy import copy from typing import Any @@ -33,10 +34,7 @@ class RestartLanguageServerTool(Tool): def apply(self) -> str: """Use this tool only on explicit user request or after confirmation. - It may be necessary to restart the language server if the user performs edits - not through Serena, so the language server state becomes outdated and further editing attempts lead to errors. - - If such editing errors happen, you should suggest using this tool. + It may be necessary to restart the language server if it hangs. """ self.agent.reset_language_server() return SUCCESS_RESULT @@ -49,24 +47,28 @@ class GetSymbolsOverviewTool(Tool): def apply(self, relative_path: str, max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH) -> str: """ - Gets an overview of the given file or directory. - For each analyzed file, we list the top-level symbols in the file (name_path, kind). - Use this tool to get a high-level understanding of the code symbols. - Calling this is often a good idea before more targeted reading, searching or editing operations on the code symbols. - Before requesting a symbol overview, it is usually a good idea to narrow down the scope of the overview - by first understanding the basic directory structure of the repository that you can get from memories - or by using the `list_dir` and `find_file` tools (or similar). + Use this tool to get a high-level understanding of the code symbols in a file. + This should be the first tool to call when you want to understand a new file, unless you already know + what you are looking for. :param relative_path: the relative path to the file or directory to get the overview of :param max_answer_chars: if the overview is longer than this number of characters, no content will be returned. Don't adjust unless there is really no other way to get the content required for the task. If the overview is too long, you should use a smaller directory instead, (e.g. a subdirectory). - :return: a JSON object mapping relative paths of all contained files to info about top-level symbols in the file (name_path, kind). + :return: a JSON object containing info about top-level symbols in the file """ symbol_retriever = self.create_language_server_symbol_retriever() - result = symbol_retriever.get_symbol_overview(relative_path) - result_json_str = json.dumps({k: [dataclasses.asdict(i) for i in l] for k, l in result.items()}) + file_path = os.path.join(self.project.project_root, relative_path) + + # The symbol overview is capable of working with both files and directories, + # but we want to ensure that the user provides a file path. + if not os.path.exists(file_path): + raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.") + if os.path.isdir(file_path): + raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") + result = symbol_retriever.get_symbol_overview(relative_path)[relative_path] + result_json_str = json.dumps([dataclasses.asdict(i) for i in result]) return self._limit_length(result_json_str, max_answer_chars) @@ -132,7 +134,7 @@ class FindSymbolTool(Tool): :param exclude_kinds: Optional. List of LSP symbol kind integers to exclude. Takes precedence over `include_kinds`. :param substring_matching: If True, use substring matching for the last segment of `name`. :param max_answer_chars: Max characters for the JSON result. If exceeded, no content is returned. - :return: JSON string: a list of symbols (with locations) matching the name. + :return: a list of symbols (with locations) matching the name. """ parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None @@ -164,10 +166,8 @@ class FindReferencingSymbolsTool(Tool): max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Finds symbols that reference the symbol at the given `name_path`. The result will contain metadata about the referencing symbols - as well as a short code snippet around the reference (unless `include_body` is True, then the short snippet will be omitted). - Note that among other kinds of references, this function can be used to find (direct) subclasses of a class, - as subclasses are referencing symbols that have the kind class. + Finds references to the symbol at the given `name_path`. The result will contain metadata about the referencing symbols + as well as a short code snippet around the reference. :param name_path: for finding the symbol to find references for, same logic as in the `find_symbol` tool. :param relative_path: the relative path to the file containing the symbol for which to find references. @@ -269,9 +269,9 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): body: str, ) -> str: """ - Inserts the given body/content before the beginning of the definition of the given symbol (via the symbol's location). - A typical use case is to insert a new class, function, method, field or variable assignment. - It also can be used to insert a new import statement before the first symbol in the file. + Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). + A typical use case is to insert a new class, function, method, field or variable assignment; or + a new import statement before the first symbol in the file. :param name_path: name path of the symbol before which to insert content (definitions in the `find_symbol` tool apply) :param relative_path: the relative path to the file containing the symbol diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 38cc19be..be78d03d 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -96,7 +96,7 @@ class ThinkAboutWhetherYouAreDoneTool(Tool): return self.prompt_factory.create_think_about_whether_you_are_done() -class SummarizeChangesTool(Tool): +class SummarizeChangesTool(Tool, ToolMarkerOptional): """ Provides instructions for summarizing the changes made to the codebase. """ From 3d8f5f7446508bd5cd68db826b66109ffa685524 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 00:14:23 +0200 Subject: [PATCH 031/220] Added chatgpt context --- .../resources/config/contexts/chatgpt.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/serena/resources/config/contexts/chatgpt.yml diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml new file mode 100644 index 00000000..c758b2c3 --- /dev/null +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -0,0 +1,28 @@ +description: A configuration specific for chatgpt, which has a limit of 30 tools and requires short descriptions. +prompt: | + You are running in desktop app context where the tools give you access to the code base as well as some + access to the file system, if configured. You interact with the user through a chat interface that is separated + from the code base. As a consequence, if you are in interactive mode, your communication with the user should + involve high-level thinking and planning as well as some summarization of any code edits that you make. + For viewing the code edits the user will view them in a separate code editor window, and the back-and-forth + between the chat and the code editor should be minimized as well as facilitated by you. + If complex changes have been made, advise the user on how to review them in the code editor. + If complex relationships that the user asked for should be visualized or explained, consider creating + a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering + options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. +excluded_tools: [] +included_optional_tools: + - switch_modes + +tool_description_overrides: + find_symbol: | + Retrieves symbols matching `name_path` in a file. + Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" + replace_regex: | + Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + Essential for multi-line replacements. + search_for_pattern: | + Flexible pattern search across codebase. Prefer symbolic operations when possible. + Uses DOTALL matching. Use non-greedy quantifiers (.*?) to avoid over-matching. + Supports file filtering via globs and code-only restriction. \ No newline at end of file From 069db44091afc90385606c72e07a5f6552b58b06 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 00:26:03 +0200 Subject: [PATCH 032/220] GH: fixing paths, hopefully cache starts working on unix --- .github/workflows/pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 91dcd610..31f4b372 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -36,8 +36,8 @@ jobs: uses: actions/cache@v3 with: path: | - ~/go/bin - ~/.cache/go-build + ${{ env.HOME }}/go/bin + ${{ env.HOME }}/.cache/go-build key: go-binaries-${{ runner.os }}-gopls-latest - name: Install gopls if: steps.cache-go-binaries.outputs.cache-hit != 'true' @@ -77,7 +77,7 @@ jobs: id: cache-language-servers uses: actions/cache@v3 with: - path: ~/.serena/language_servers/static + path: ${{ env.HOME }}/.serena/language_servers/static key: language-servers-${{ runner.os }}-v1 restore-keys: | language-servers-${{ runner.os }}- From 2087bd3d1b1c521af025234d69df60306309078d Mon Sep 17 00:00:00 2001 From: Yangze Luo Date: Sun, 3 Aug 2025 13:26:07 +0800 Subject: [PATCH 033/220] Add swift lsp support --- .github/workflows/pytest.yml | 34 +- .gitignore | 4 + README.md | 1 + pyproject.toml | 3 +- .../language_servers/sourcekit_lsp.py | 561 ++++++++++++++++++ src/solidlsp/ls.py | 5 + src/solidlsp/ls_config.py | 3 + .../repos/swift/test_repo/Package.swift | 16 + .../repos/swift/test_repo/src/main.swift | 62 ++ .../repos/swift/test_repo/src/utils.swift | 20 + test/solidlsp/swift/test_swift_basic.py | 228 +++++++ 11 files changed, 933 insertions(+), 4 deletions(-) create mode 100644 src/solidlsp/language_servers/sourcekit_lsp.py create mode 100644 test/resources/repos/swift/test_repo/Package.swift create mode 100644 test/resources/repos/swift/test_repo/src/main.swift create mode 100644 test/resources/repos/swift/test_repo/src/utils.swift create mode 100644 test/solidlsp/swift/test_swift_basic.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 91dcd610..dc4be450 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -27,7 +27,7 @@ jobs: python-version: "${{ matrix.python-version }}" - uses: actions/setup-go@v5 with: - go-version: '>=1.17.0' + go-version: ">=1.17.0" # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps @@ -47,8 +47,8 @@ jobs: if: runner.os != 'Windows' uses: erlef/setup-beam@v1 with: - elixir-version: '1.18.4' - otp-version: '26.1' + elixir-version: "1.18.4" + otp-version: "26.1" - name: Prepare java uses: actions/setup-java@v3 @@ -64,6 +64,34 @@ jobs: with: terraform_version: "1.5.0" terraform_wrapper: false + - name: Install Swift with swiftly (macOS) + if: runner.os == 'macOS' + run: | + echo "=== Installing swiftly on macOS ===" + curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg && \ + installer -pkg swiftly.pkg -target CurrentUserHomeDirectory && \ + ~/.swiftly/bin/swiftly init --quiet-shell-followup && \ + . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" && \ + hash -r + swiftly install --use 6.1.2 + swiftly use 6.1.2 + echo "~/.swiftly/bin" >> $GITHUB_PATH + echo "Swiftly installed successfully" + - name: Install Swift with swiftly (Ubuntu) + if: runner.os == 'Linux' + run: | + echo "=== Installing swiftly on Ubuntu ===" + curl -O https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz && \ + tar zxf swiftly-$(uname -m).tar.gz && \ + ./swiftly init --quiet-shell-followup && \ + . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ + hash -r + swiftly install --use 6.1.2 + swiftly use 6.1.2 + sudo apt-get -y install libcurl4-openssl-dev + echo "=== Adding Swift toolchain to PATH ===" + echo "$HOME/.local/share/swiftly/bin" >> $GITHUB_PATH + echo "Swiftly installed successfully!" - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/.gitignore b/.gitignore index 64c7e71e..fdbd1c9f 100644 --- a/.gitignore +++ b/.gitignore @@ -227,3 +227,7 @@ tmp/ /test/resources/repos/elixir/test_repo/deps # Exception: Don't ignore Elixir test repository lib directory (contains source code) !/test/resources/repos/elixir/test_repo/lib + +# Swift +/test/resources/repos/swift/test_repo/.build +/test/resources/repos/swift/test_repo/.swiftpm diff --git a/README.md b/README.md index 1b6777cb..14a7d839 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ With Serena, we provide * Elixir (Requires NextLS and Elixir install; **Windows not supported** - Next LS does not provide Windows binaries) * Clojure * C/C++ (You may experience issues with finding references, we are working on it) + * Swift (requires Swift and sourcekit-lsp to be installed) * indirect support (may require some code changes/manual installation) for: * Ruby (untested) * Kotlin (untested) diff --git a/pyproject.toml b/pyproject.toml index 8872384c..fc20b70f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ explicit = true [project.scripts] serena = "serena.cli:top_level" serena-mcp-server = "serena.cli:start_mcp_server" -index-project = "serena.cli:index_project" # deprecated +index-project = "serena.cli:index_project" # deprecated [project.license] text = "MIT" @@ -259,6 +259,7 @@ markers = [ "csharp: language server running for C#", "elixir: language server running for Elixir", "terraform: language server running for Terraform", + "swift: language server running for Swift", "snapshot: snapshot tests for symbolic editing operations", ] diff --git a/src/solidlsp/language_servers/sourcekit_lsp.py b/src/solidlsp/language_servers/sourcekit_lsp.py new file mode 100644 index 00000000..dac6cfe4 --- /dev/null +++ b/src/solidlsp/language_servers/sourcekit_lsp.py @@ -0,0 +1,561 @@ +import logging +import os +import pathlib +import subprocess +import threading + +from overrides import override + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler import lsp_types +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings +from solidlsp.lsp_protocol_handler import lsp_types as LSPTypes + + +class SourceKitLSP(SolidLanguageServer): + """ + Provides Swift specific instantiation of the LanguageServer class using sourcekit-lsp. + """ + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + # For Swift projects, we should ignore: + # - .build: Swift Package Manager build artifacts + # - .swiftpm: Swift Package Manager metadata + # - node_modules: if the project has JavaScript components + # - dist/build: common output directories + return super().is_ignored_dirname(dirname) or dirname in [".build", ".swiftpm", "node_modules", "dist", "build"] + + @staticmethod + def _get_swift_version(): + """Get the installed Swift version or None if not found.""" + try: + result = subprocess.run(["swift", "--version"], capture_output=True, text=True, check=False) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + return None + return None + + @staticmethod + def _get_sourcekit_lsp_version(): + """Get the installed sourcekit-lsp version or None if not found.""" + try: + result = subprocess.run(["sourcekit-lsp", "-h"], capture_output=True, text=True, check=False) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + return None + return None + + @staticmethod + def _setup_runtime_dependency(): + """ + Check if required Swift runtime dependencies are available. + Raises RuntimeError with helpful message if dependencies are missing. + """ + swift_version = SourceKitLSP._get_swift_version() + if not swift_version: + raise RuntimeError( + "Swift is not installed. Please install Swift from https://swift.org/download/ and make sure it is added to your PATH." + ) + + sourcekit_lsp_version = SourceKitLSP._get_sourcekit_lsp_version() + if not sourcekit_lsp_version: + raise RuntimeError( + "Found a Swift version but sourcekit-lsp is not installed.\n" + "Please install sourcekit-lsp as described in https://github.com/apple/sourcekit-lsp#installation\n\n" + "After installation, make sure it is added to your PATH." + ) + + return True + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + self._setup_runtime_dependency() + + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd="sourcekit-lsp", cwd=repository_root_path), + "swift", + solidlsp_settings, + ) + self.server_ready = threading.Event() + self.request_id = 0 + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Swift Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + + initialize_params = { + "capabilities": { + "general": { + "markdown": { + "parser": "marked", + "version": "1.1.0" + }, + "positionEncodings": [ + "utf-16" + ], + "regularExpressions": { + "engine": "ECMAScript", + "version": "ES2020" + }, + "staleRequestSupport": { + "cancel": True, + "retryOnContentModified": [ + "textDocument/semanticTokens/full", + "textDocument/semanticTokens/range", + "textDocument/semanticTokens/full/delta" + ] + } + }, + "notebookDocument": { + "synchronization": { + "dynamicRegistration": True, + "executionSummarySupport": True + } + }, + "textDocument": { + "callHierarchy": { + "dynamicRegistration": True + }, + "codeAction": { + "codeActionLiteralSupport": { + "codeActionKind": { + "valueSet": [ + "", + "quickfix", + "refactor", + "refactor.extract", + "refactor.inline", + "refactor.rewrite", + "source", + "source.organizeImports" + ] + } + }, + "dataSupport": True, + "disabledSupport": True, + "dynamicRegistration": True, + "honorsChangeAnnotations": True, + "isPreferredSupport": True, + "resolveSupport": { + "properties": [ + "edit" + ] + } + }, + "codeLens": { + "dynamicRegistration": True + }, + "colorProvider": { + "dynamicRegistration": True + }, + "completion": { + "completionItem": { + "commitCharactersSupport": True, + "deprecatedSupport": True, + "documentationFormat": [ + "markdown", + "plaintext" + ], + "insertReplaceSupport": True, + "insertTextModeSupport": { + "valueSet": [ + 1, + 2 + ] + }, + "labelDetailsSupport": True, + "preselectSupport": True, + "resolveSupport": { + "properties": [ + "documentation", + "detail", + "additionalTextEdits" + ] + }, + "snippetSupport": True, + "tagSupport": { + "valueSet": [ + 1 + ] + } + }, + "completionItemKind": { + "valueSet": [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 + ] + }, + "completionList": { + "itemDefaults": [ + "commitCharacters", + "editRange", + "insertTextFormat", + "insertTextMode", + "data" + ] + }, + "contextSupport": True, + "dynamicRegistration": True, + "insertTextMode": 2 + }, + "declaration": { + "dynamicRegistration": True, + "linkSupport": True + }, + "definition": { + "dynamicRegistration": True, + "linkSupport": True + }, + "diagnostic": { + "dynamicRegistration": True, + "relatedDocumentSupport": False + }, + "documentHighlight": { + "dynamicRegistration": True + }, + "documentLink": { + "dynamicRegistration": True, + "tooltipSupport": True + }, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "labelSupport": True, + "symbolKind": { + "valueSet": [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 + ] + }, + "tagSupport": { + "valueSet": [ + 1 + ] + } + }, + "foldingRange": { + "dynamicRegistration": True, + "foldingRange": { + "collapsedText": False + }, + "foldingRangeKind": { + "valueSet": [ + "comment", + "imports", + "region" + ] + }, + "lineFoldingOnly": True, + "rangeLimit": 5000 + }, + "formatting": { + "dynamicRegistration": True + }, + "hover": { + "contentFormat": [ + "markdown", + "plaintext" + ], + "dynamicRegistration": True + }, + "implementation": { + "dynamicRegistration": True, + "linkSupport": True + }, + "inlayHint": { + "dynamicRegistration": True, + "resolveSupport": { + "properties": [ + "tooltip", + "textEdits", + "label.tooltip", + "label.location", + "label.command" + ] + } + }, + "inlineValue": { + "dynamicRegistration": True + }, + "linkedEditingRange": { + "dynamicRegistration": True + }, + "onTypeFormatting": { + "dynamicRegistration": True + }, + "publishDiagnostics": { + "codeDescriptionSupport": True, + "dataSupport": True, + "relatedInformation": True, + "tagSupport": { + "valueSet": [ + 1, + 2 + ] + }, + "versionSupport": False + }, + "rangeFormatting": { + "dynamicRegistration": True, + "rangesSupport": True + }, + "references": { + "dynamicRegistration": True + }, + "rename": { + "dynamicRegistration": True, + "honorsChangeAnnotations": True, + "prepareSupport": True, + "prepareSupportDefaultBehavior": 1 + }, + "selectionRange": { + "dynamicRegistration": True + }, + "semanticTokens": { + "augmentsSyntaxTokens": True, + "dynamicRegistration": True, + "formats": [ + "relative" + ], + "multilineTokenSupport": False, + "overlappingTokenSupport": False, + "requests": { + "full": { + "delta": True + }, + "range": True + }, + "serverCancelSupport": True, + "tokenModifiers": [ + "declaration", + "definition", + "readonly", + "static", + "deprecated", + "abstract", + "async", + "modification", + "documentation", + "defaultLibrary" + ], + "tokenTypes": [ + "namespace", + "type", + "class", + "enum", + "interface", + "struct", + "typeParameter", + "parameter", + "variable", + "property", + "enumMember", + "event", + "function", + "method", + "macro", + "keyword", + "modifier", + "comment", + "string", + "number", + "regexp", + "operator", + "decorator" + ] + }, + "signatureHelp": { + "contextSupport": True, + "dynamicRegistration": True, + "signatureInformation": { + "activeParameterSupport": True, + "documentationFormat": [ + "markdown", + "plaintext" + ], + "parameterInformation": { + "labelOffsetSupport": True + } + } + }, + "synchronization": { + "didSave": True, + "dynamicRegistration": True, + "willSave": True, + "willSaveWaitUntil": True + }, + "typeDefinition": { + "dynamicRegistration": True, + "linkSupport": True + }, + "typeHierarchy": { + "dynamicRegistration": True + } + }, + "window": { + "showDocument": { + "support": True + }, + "showMessage": { + "messageActionItem": { + "additionalPropertiesSupport": True + } + }, + "workDoneProgress": True + }, + "workspace": { + "applyEdit": True, + "codeLens": { + "refreshSupport": True + }, + "configuration": True, + "diagnostics": { + "refreshSupport": True + }, + "didChangeConfiguration": { + "dynamicRegistration": True + }, + "didChangeWatchedFiles": { + "dynamicRegistration": True, + "relativePatternSupport": True + }, + "executeCommand": { + "dynamicRegistration": True + }, + "fileOperations": { + "didCreate": True, + "didDelete": True, + "didRename": True, + "dynamicRegistration": True, + "willCreate": True, + "willDelete": True, + "willRename": True + }, + "foldingRange": { + "refreshSupport": True + }, + "inlayHint": { + "refreshSupport": True + }, + "inlineValue": { + "refreshSupport": True + }, + "semanticTokens": { + "refreshSupport": False + }, + "symbol": { + "dynamicRegistration": True, + "resolveSupport": { + "properties": [ + "location.range" + ] + }, + "symbolKind": { + "valueSet": [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 + ] + }, + "tagSupport": { + "valueSet": [ + 1 + ] + } + }, + "workspaceEdit": { + "changeAnnotationSupport": { + "groupsOnLabel": True + }, + "documentChanges": True, + "failureHandling": "textOnlyTransactional", + "normalizesLineEndings": True, + "resourceOperations": [ + "create", + "rename", + "delete" + ] + }, + "workspaceFolders": True + } + }, + "clientInfo": { + "name": "Visual Studio Code", + "version": "1.102.2" + }, + "initializationOptions": { + "backgroundIndexing": True, + "backgroundPreparationMode": "enabled", + "textDocument/codeLens": { + "supportedCommands": { + "swift.debug": "swift.debug", + "swift.run": "swift.run" + } + }, + "window/didChangeActiveDocument": True, + "workspace/getReferenceDocument": True, + "workspace/peekDocuments": True + }, + "locale": "en", + "processId": os.getpid(), + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + + return initialize_params + + def _start_server(self): + """Start sourcekit-lsp server process""" + + def register_capability_handler(_params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + + def do_nothing(_params): + return + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting sourcekit-lsp server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + + capabilities = init_response["capabilities"] + self.logger.log(f"SourceKit LSP capabilities: {list(capabilities.keys())}", logging.INFO) + + assert "textDocumentSync" in capabilities, "textDocumentSync capability missing" + assert "definitionProvider" in capabilities, "definitionProvider capability missing" + + self.server.notify.initialized({}) + self.completions_available.set() + + # sourcekit-lsp server is typically ready immediately after initialization + self.server_ready.set() + self.server_ready.wait() \ No newline at end of file diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index ded2645a..5e82b2ef 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -226,6 +226,11 @@ class SolidLanguageServer(ABC): ls = TerraformLS(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + elif config.code_language == Language.SWIFT: + from solidlsp.language_servers.sourcekit_lsp import SourceKitLSP + + ls = SourceKitLSP(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + else: logger.log(f"Language {config.code_language} is not supported", logging.ERROR) raise SolidLSPException(f"Language {config.code_language} is not supported") diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index efb55a87..363027c8 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -42,6 +42,7 @@ class Language(str, Enum): CLOJURE = "clojure" ELIXIR = "elixir" TERRAFORM = "terraform" + SWIFT = "swift" # Experimental or deprecated Language Servers TYPESCRIPT_VTS = "typescript_vts" """Use the typescript language server through the natively bundled vscode extension via https://github.com/yioneko/vtsls""" @@ -103,6 +104,8 @@ class Language(str, Enum): return FilenameMatcher("*.ex", "*.exs") case self.TERRAFORM: return FilenameMatcher("*.tf", "*.tfvars", "*.tfstate") + case self.SWIFT: + return FilenameMatcher("*.swift") case _: raise ValueError(f"Unhandled language: {self}") diff --git a/test/resources/repos/swift/test_repo/Package.swift b/test/resources/repos/swift/test_repo/Package.swift new file mode 100644 index 00000000..13a02704 --- /dev/null +++ b/test/resources/repos/swift/test_repo/Package.swift @@ -0,0 +1,16 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "test_repo", + products: [ + .library( + name: "test_repo", + targets: ["test_repo"]), + ], + targets: [ + .target( + name: "test_repo", + dependencies: []), + ] +) \ No newline at end of file diff --git a/test/resources/repos/swift/test_repo/src/main.swift b/test/resources/repos/swift/test_repo/src/main.swift new file mode 100644 index 00000000..84630774 --- /dev/null +++ b/test/resources/repos/swift/test_repo/src/main.swift @@ -0,0 +1,62 @@ +import Foundation + +// Main entry point +func main() { + let calculator = Calculator() + let result = calculator.add(5, 3) + print("Result: \(result)") + + let user = User(name: "Alice", age: 30) + user.greet() + + let area = Utils.calculateArea(radius: 5.0) + print("Circle area: \(area)") +} + +class Calculator { + func add(_ a: Int, _ b: Int) -> Int { + return a + b + } + + func multiply(_ a: Int, _ b: Int) -> Int { + return a * b + } +} + +struct User { + let name: String + let age: Int + + func greet() { + print("Hello, my name is \(name) and I am \(age) years old.") + } + + func isAdult() -> Bool { + return age >= 18 + } +} + +enum Status { + case active + case inactive + case pending +} + +protocol Drawable { + func draw() +} + +class Circle: Drawable { + let radius: Double + + init(radius: Double) { + self.radius = radius + } + + func draw() { + print("Drawing a circle with radius \(radius)") + } +} + +// Call main +main() \ No newline at end of file diff --git a/test/resources/repos/swift/test_repo/src/utils.swift b/test/resources/repos/swift/test_repo/src/utils.swift new file mode 100644 index 00000000..cf1422c7 --- /dev/null +++ b/test/resources/repos/swift/test_repo/src/utils.swift @@ -0,0 +1,20 @@ +import Foundation + +public struct Utils { + public static func formatDate(_ date: Date) -> String { + let formatter = DateFormatter() + formatter.dateStyle = .medium + return formatter.string(from: date) + } + + public static func calculateArea(radius: Double) -> Double { + return Double.pi * radius * radius + } +} + +public extension String { + func isValidEmail() -> Bool { + let emailRegex = "[A-Z0-9a-z._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}" + return NSPredicate(format: "SELF MATCHES %@", emailRegex).evaluate(with: self) + } +} \ No newline at end of file diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py new file mode 100644 index 00000000..464553e9 --- /dev/null +++ b/test/solidlsp/swift/test_swift_basic.py @@ -0,0 +1,228 @@ +""" +Basic integration tests for the Swift language server functionality. + +These tests validate the functionality of the language server APIs +like request_references using the Swift test repository. +""" + +import os +import sys +import time +import platform + +import pytest + +from serena.project import Project +from serena.text_utils import LineType +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +# Skip Swift tests on Windows due to complex GitHub Actions configuration +WINDOWS_SKIP = platform.system() == "Windows" +WINDOWS_SKIP_REASON = "GitHub Actions configuration for Swift on Windows is complex, skipping for now." + +pytestmark = [pytest.mark.swift, pytest.mark.skipif(WINDOWS_SKIP, reason=WINDOWS_SKIP_REASON)] + +class TestSwiftLanguageServerBasics: + """Test basic functionality of the Swift language server.""" + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_goto_definition_calculator_class(self, language_server: SolidLanguageServer) -> None: + """Test goto_definition on Calculator class usage.""" + file_path = os.path.join("src", "main.swift") + + # Find the Calculator usage at line 5: let calculator = Calculator() + # Position should be at the "Calculator()" call + definitions = language_server.request_definition(file_path, 4, 23) # Position at Calculator() call + assert isinstance(definitions, list), "Definitions should be a list" + assert len(definitions) > 0, "Should find definition for Calculator class" + + # Verify the definition points to the Calculator class definition + calculator_def = definitions[0] + assert calculator_def.get("uri", "").endswith("main.swift"), "Definition should be in main.swift" + + # The Calculator class is defined starting at line 16 + start_line = calculator_def.get("range", {}).get("start", {}).get("line") + assert start_line == 15, f"Calculator class definition should be at line 16, got {start_line + 1}" + + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_goto_definition_user_struct(self, language_server: SolidLanguageServer) -> None: + """Test goto_definition on User struct usage.""" + file_path = os.path.join("src", "main.swift") + + # Find the User usage at line 9: let user = User(name: "Alice", age: 30) + # Position should be at the "User(...)" call + definitions = language_server.request_definition(file_path, 8, 18) # Position at User(...) call + assert isinstance(definitions, list), "Definitions should be a list" + assert len(definitions) > 0, "Should find definition for User struct" + + # Verify the definition points to the User struct definition + user_def = definitions[0] + assert user_def.get("uri", "").endswith("main.swift"), "Definition should be in main.swift" + + # The User struct is defined starting at line 26 + start_line = user_def.get("range", {}).get("start", {}).get("line") + assert start_line == 25, f"User struct definition should be at line 26, got {start_line + 1}" + + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_goto_definition_calculator_method(self, language_server: SolidLanguageServer) -> None: + """Test goto_definition on Calculator method usage.""" + file_path = os.path.join("src", "main.swift") + + # Find the add method usage at line 6: let result = calculator.add(5, 3) + # Position should be at the "add" method call + definitions = language_server.request_definition(file_path, 5, 28) # Position at add method call + assert isinstance(definitions, list), "Definitions should be a list" + + # Verify the definition points to the add method definition + add_def = definitions[0] + assert add_def.get("uri", "").endswith("main.swift"), "Definition should be in main.swift" + + # The add method is defined starting at line 17 + start_line = add_def.get("range", {}).get("start", {}).get("line") + assert start_line == 16, f"add method definition should be at line 17, got {start_line + 1}" + + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_goto_definition_cross_file(self, language_server: SolidLanguageServer) -> None: + """Test goto_definition across files - Utils struct.""" + utils_file = os.path.join("src", "utils.swift") + + # First, let's check if Utils is used anywhere (it might not be in this simple test) + # We'll test goto_definition on Utils struct itself + symbols = language_server.request_document_symbols(utils_file) + utils_symbol = next((s for s in symbols[0] if s.get("name") == "Utils"), None) + + sel_start = utils_symbol["selectionRange"]["start"] + definitions = language_server.request_definition(utils_file, sel_start["line"], sel_start["character"]) + assert isinstance(definitions, list), "Definitions should be a list" + + # Should find the Utils struct definition itself + utils_def = definitions[0] + assert utils_def.get("uri", "").endswith("utils.swift"), "Definition should be in utils.swift" + + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_request_references_calculator_class(self, language_server: SolidLanguageServer) -> None: + # SourceKit LSP needs a short initialization period after startup + # before it can provide accurate reference information. This sleep + # prevents race conditions where references might not be available yet. + time.sleep(5) + """Test request_references on the Calculator class.""" + # Get references to the Calculator class in main.swift + file_path = os.path.join("src", "main.swift") + symbols = language_server.request_document_symbols(file_path) + + calculator_symbol = next((s for s in symbols[0] if s.get("name") == "Calculator"), None) + + sel_start = calculator_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert isinstance(references, list), "References should be a list" + assert len(references) > 0, "Calculator class should be referenced" + + # Validate that Calculator is referenced in the main function + calculator_refs = [ref for ref in references if ref.get("uri", "").endswith("main.swift")] + assert len(calculator_refs) > 0, "Calculator class should be referenced in main.swift" + + # Check that one reference is at line 5 (let calculator = Calculator()) + line_5_refs = [ref for ref in calculator_refs if ref.get("range", {}).get("start", {}).get("line") == 4] + assert len(line_5_refs) > 0, "Calculator should be referenced at line 5" + + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_request_references_user_struct(self, language_server: SolidLanguageServer) -> None: + """Test request_references on the User struct.""" + # Get references to the User struct in main.swift + file_path = os.path.join("src", "main.swift") + symbols = language_server.request_document_symbols(file_path) + + user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) + + sel_start = user_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert isinstance(references, list), "References should be a list" + + # Validate that User is referenced in the main function + user_refs = [ref for ref in references if ref.get("uri", "").endswith("main.swift")] + assert len(user_refs) > 0, "User struct should be referenced in main.swift" + + # Check that one reference is at line 9 (let user = User(...)) + line_9_refs = [ref for ref in user_refs if ref.get("range", {}).get("start", {}).get("line") == 8] + assert len(line_9_refs) > 0, "User should be referenced at line 9" + + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) + def test_request_references_utils_struct(self, language_server: SolidLanguageServer) -> None: + """Test request_references on the Utils struct.""" + # Get references to the Utils struct in utils.swift + file_path = os.path.join("src", "utils.swift") + symbols = language_server.request_document_symbols(file_path) + utils_symbol = next((s for s in symbols[0] if s.get("name") == "Utils"), None) + if not utils_symbol or "selectionRange" not in utils_symbol: + raise AssertionError("Utils symbol or its selectionRange not found") + sel_start = utils_symbol["selectionRange"]["start"] + references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert isinstance(references, list), "References should be a list" + assert len(references) > 0, "Utils struct should be referenced" + + # Validate that Utils is referenced in main.swift + utils_refs = [ref for ref in references if ref.get("uri", "").endswith("main.swift")] + assert len(utils_refs) > 0, "Utils struct should be referenced in main.swift" + + # Check that one reference is at line 12 (Utils.calculateArea call) + line_12_refs = [ref for ref in utils_refs if ref.get("range", {}).get("start", {}).get("line") == 11] + assert len(line_12_refs) > 0, "Utils should be referenced at line 12" + + + +class TestSwiftProjectBasics: + @pytest.mark.parametrize("project", [Language.SWIFT], indirect=True) + def test_retrieve_content_around_line(self, project: Project) -> None: + """Test retrieve_content_around_line functionality with various scenarios.""" + file_path = os.path.join("src", "main.swift") + + # Scenario 1: Find Calculator class definition + calculator_line = None + for line_num in range(1, 50): # Search first 50 lines + try: + line_content = project.retrieve_content_around_line(file_path, line_num) + if line_content.lines and "class Calculator" in line_content.lines[0].line_content: + calculator_line = line_num + break + except: + continue + + assert calculator_line is not None, "Calculator class not found" + line_calc = project.retrieve_content_around_line(file_path, calculator_line) + assert len(line_calc.lines) == 1 + assert "class Calculator" in line_calc.lines[0].line_content + assert line_calc.lines[0].line_number == calculator_line + assert line_calc.lines[0].match_type == LineType.MATCH + + # Scenario 2: Context above and below Calculator class + with_context_around_calculator = project.retrieve_content_around_line(file_path, calculator_line, 2, 2) + assert len(with_context_around_calculator.lines) == 5 + assert "class Calculator" in with_context_around_calculator.matched_lines[0].line_content + assert with_context_around_calculator.num_matched_lines == 1 + + # Scenario 3: Search for struct definitions + struct_pattern = r"struct\s+\w+" + matches = project.search_source_files_for_pattern(struct_pattern) + assert len(matches) > 0, "Should find struct definitions" + # Should find User struct + user_matches = [m for m in matches if "User" in str(m)] + assert len(user_matches) > 0, "Should find User struct" + + # Scenario 4: Search for class definitions + class_pattern = r"class\s+\w+" + matches = project.search_source_files_for_pattern(class_pattern) + assert len(matches) > 0, "Should find class definitions" + # Should find Calculator and Circle classes + calculator_matches = [m for m in matches if "Calculator" in str(m)] + circle_matches = [m for m in matches if "Circle" in str(m)] + assert len(calculator_matches) > 0, "Should find Calculator class" + assert len(circle_matches) > 0, "Should find Circle class" + + # Scenario 5: Search for enum definitions + enum_pattern = r"enum\s+\w+" + matches = project.search_source_files_for_pattern(enum_pattern) + assert len(matches) > 0, "Should find enum definitions" + # Should find Status enum + status_matches = [m for m in matches if "Status" in str(m)] + assert len(status_matches) > 0, "Should find Status enum" \ No newline at end of file From 7990c3f30d416321e3c69d57fe165c1be86d6d01 Mon Sep 17 00:00:00 2001 From: Kazuki Hanai Date: Sun, 3 Aug 2025 19:24:42 +0900 Subject: [PATCH 034/220] Fix: Add HTML escaping to dashboard log messages Escape special HTML characters in log messages to prevent XSS vulnerabilities and ensure proper display of code snippets containing HTML tags. --- src/serena/resources/dashboard/dashboard.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index b57226aa..318d0c16 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -1,5 +1,6 @@ class LogMessage { constructor(message, toolNames) { + message = this.escapeHtml(message); const logLevel = this.determineLogLevel(message); const highlightedMessage = this.highlightToolNames(message, toolNames); this.$elem = $('
').addClass('log-' + logLevel).html(highlightedMessage + '\n'); @@ -27,6 +28,21 @@ class LogMessage { }); return highlightedMessage; } + + escapeHtml (convertString) { + if (typeof convertString !== 'string') return convertString; + + const patterns = { + '<' : '<', + '>' : '>', + '&' : '&', + '"' : '"', + '\'' : ''', + '`' : '`' + }; + + return convertString.replace(/[<>&"'`]/g, match => patterns[match]); + }; } class Dashboard { From 574eb7b327a7c765c3cfca5d74b59f811140546b Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 13:35:33 +0200 Subject: [PATCH 035/220] GH: make sure cached dirs exist --- .github/workflows/pytest.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 31f4b372..3eeac6be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -28,6 +28,12 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '>=1.17.0' + - name: Ensure cached directory exist before calling cache-related actions + shell: bash + run: | + mkdir -p $HOME/.serena/language_servers/static + mkdir -p $HOME/.cache/go-build + mkdir -p $HOME/go/bin # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps @@ -36,8 +42,8 @@ jobs: uses: actions/cache@v3 with: path: | - ${{ env.HOME }}/go/bin - ${{ env.HOME }}/.cache/go-build + ~/go/bin + ~/.cache/go-build key: go-binaries-${{ runner.os }}-gopls-latest - name: Install gopls if: steps.cache-go-binaries.outputs.cache-hit != 'true' @@ -49,7 +55,6 @@ jobs: with: elixir-version: '1.18.4' otp-version: '26.1' - - name: Prepare java uses: actions/setup-java@v3 with: @@ -77,7 +82,7 @@ jobs: id: cache-language-servers uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.serena/language_servers/static + path: ~/.serena/language_servers/static key: language-servers-${{ runner.os }}-v1 restore-keys: | language-servers-${{ runner.os }}- From 658e1f5f91923d77110699825a099534e38b09d3 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 14:45:43 +0200 Subject: [PATCH 036/220] Updated readme to reflect newer developments --- README.md | 90 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 1b6777cb..112346e0 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@

* :rocket: Serena is a powerful **coding agent toolkit** capable of turning an LLM into a fully-featured agent that works **directly on your codebase**. + Unlike most other tools, it is not tied to an LLM, framework or an interface, making it easy to use it in a variety of ways. * :wrench: Serena provides essential **semantic code retrieval and editing tools** that are akin to an IDE's capabilities, extracting code entities at the symbol level and exploiting relational structure. When combined with an existing coding agent, these tools greatly enhance (token) efficiency. -* :free: Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge. +* :free: Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge. ### Demonstration @@ -45,10 +46,9 @@ Serena can be integrated with an LLM in several ways: * Claude Code and Claude Desktop, * IDEs like VSCode, Cursor or IntelliJ, * Extensions like Cline or Roo Code - * and many others, including [the ChatGPT app soon](https://x.com/OpenAIDevs/status/1904957755829481737) - * by using **Agno – the model-agnostic agent framework**. - Serena's Agno-based agent allows you to turn virtually any LLM into a coding agent, whether it's provided by Google, OpenAI or Anthropic (with a paid API key) - or a free model provided by Ollama, Together or Anyscale. + * Local clients like [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp), [Jan](https://jan.ai/docs/mcp-examples/browser/browserbase#enable-mcp), Agno and others + * by using [mcpo to connect it to ChatGPT](docs/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling. + * by using in your custom agent beyond MCP, illustrated in our [Agno tooll integration](#agno-agent). * by incorporating Serena's tools into an agent framework of your choice. Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework. @@ -96,22 +96,27 @@ implementation. - [What Can I Use Serena For?](#what-can-i-use-serena-for) + * [As MCP Server](#as-mcp-server) + * [As Library](#as-library) - [Free Coding Agents with Serena](#free-coding-agents-with-serena) + * [Note](#note) - [Quick Start](#quick-start) * [Running the Serena MCP Server](#running-the-serena-mcp-server) + [Usage](#usage) - * [Local Installation](#local-installation) - [Using uvx](#using-uvx) + * [Local Installation](#local-installation) - [Using Docker (Experimental)](#using-docker-experimental) + [SSE Mode](#sse-mode) + [Command-Line Arguments](#command-line-arguments) * [Configuration](#configuration) * [Project Activation & Indexing](#project-activation--indexing) * [Claude Code](#claude-code) + * [Other Terminal-Based Clients](#other-terminal-based-clients) * [Claude Desktop](#claude-desktop) - * [Other MCP Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)](#other-mcp-clients-cline-roo-code-cursor-windsurf-etc) - * [Agno Agent](#agno-agent) - * [Other Agent Frameworks](#other-agent-frameworks) + * [MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)](#mcp-coding-clients-cline-roo-code-cursor-windsurf-etc) + * [Local GUIs and Frameworks](#local-guis-and-frameworks) + * [Custom Integrations](#custom-integrations) + + [Other Agent Frameworks](#other-agent-frameworks) - [Detailed Usage and Recommendations](#detailed-usage-and-recommendations) * [Tool Execution](#tool-execution) + [Shell Execution and Editing Tools](#shell-execution-and-editing-tools) @@ -142,12 +147,16 @@ implementation. ## What Can I Use Serena For? +Serena is a toolkit, which means it can be used in various ways, depending on your needs and preferences. +It can be used as a highly configurable MCP server (application), as a toolkit for building agents (library), or +as a framework for building custom integrations based on language servers (the solidlsp package). + +### As MCP Server You can use Serena for any coding tasks – whether it is focussed on analysis, planning, designing new components or refactoring existing ones. Since Serena's tools allow an LLM to close the cognitive perception-action loop, agents based on Serena can autonomously carry out coding tasks from start to finish – -from the initial analysis to the implementation, testing and, finally, the version -control system commit. +from the initial analysis to the implementation, testing and, finally, the commit. Serena can read, write and execute code, read logs and the terminal output. While we do not necessarily encourage it, "vibe coding" is certainly possible, and if you @@ -155,11 +164,25 @@ want to almost feel like "the code no longer exists", you may find Serena even more adequate for vibing than an agent inside an IDE (since you will have a separate GUI that really lets you forget). +The application layer of Serena is highly configurable, so you can adapt it to your needs. +You can select the tools you want to use, adjust the prompts and even the tool descriptions +without writing any code. Serena comes with a CLI to facilitate this. + +See the sections on [configuration](#configuration) and [modes and contexts](#modes-and-contexts) +for more information on how to adapt Serena to your needs, +and explore the cli with `uvx --from git+https://github.com/oraios/serena serena --help`. + +### As Library +While the lower level interfaces should not be considered as stable yet and have no extensive documentation, +we do develop them with generality and ease of use in mind. +As a result, you can use Serena as a library to build your own agents using the provided tools +and language server capabilities. + + ## Free Coding Agents with Serena Even the free tier of Anthropic's Claude has support for MCP Servers, so you can use Serena with Claude for free. -Presumably, the same will soon be possible with ChatGPT Desktop once support for MCP servers is added. -Through Agno, you furthermore have the option to use Serena with a free/open-weights model. +You can also connect it to ChatGPT and turn it into a coding agent with the help of mcpo, see [here](docs/serena_on_chatgpt.md). Serena is [Oraios AI](https://oraios-ai.de/)'s contribution to the developer community. We use it ourselves on a regular basis. @@ -169,6 +192,12 @@ IDE-based subscriptions (such as Windsurf or Cursor) that forced us to keep purc The substantial API costs incurred by tools like Claude Code, Cline, Aider and other API-based tools are similarly unattractive. We thus built Serena with the prospect of being able to cancel most other subscriptions. +### Note + +We had built Serena before Claude Code was included in Antrhopic's pro tier. Now the cost saving aspect is less important, +but Serena is still valuable as its tools are more powerful than those of Claude Code or other agents that we are aware of (at least for now), so many users +find value in using Serena together with Claude Code or similar coding agents. + ## Quick Start Serena can be used in various ways, below you will find instructions for selected integrations. @@ -360,6 +389,16 @@ claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena in your config. Note that you may have to make Claude read the instructions you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. +### Other Terminal-Based Clients + +There are many terminal-based coding assistants that support MCP servers, such as [Codex](https://github.com/openai/codex?tab=readme-ov-file#model-context-protocol-mcp), +[Gemini-CLI](https://github.com/google-gemini/gemini-cli), [Qwen3-Coder](https://github.com/QwenLM/Qwen3-Coder), +[rovodev](https://community.atlassian.com/forums/Rovo-for-Software-Teams-Beta/Introducing-Rovo-Dev-CLI-AI-Powered-Development-in-your-terminal/ba-p/3043623), +the [OpenHands CLI](https://docs.all-hands.dev/usage/how-to/cli-mode) and [opencode](https://github.com/sst/opencode). + +They generally benefit from the symbolic tools provided by Serena. You might want to customize some aspects of Serena +by writing your own context, modes or prompts to adjust it to your workflow, to other MCP servers you are using, and to +the client's internal capabilities. ### Claude Desktop @@ -421,7 +460,7 @@ After restarting, you should see Serena's tools in your chat interface (notice t For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user). -### Other MCP Clients (Cline, Roo-Code, Cursor, Windsurf, etc.) +### MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.) Being an MCP Server, Serena can be included in any MCP Client. The same configuration as above, perhaps with small client-specific modifications, should work. Most of the popular @@ -437,16 +476,21 @@ e.g., for one of the following reasons: 2. You are on Linux and don't want to use the [community-created Claude Desktop](https://github.com/aaddrick/claude-desktop-debian). 3. You want tighter integration of Serena into your IDE and don't mind paying for that. -### Agno Agent +### Local GUIs and Frameworks + +Over the last months, several technologies have emerged that allow you to run a powerful local GUI +and connect it to an MCP server. They will work with Serena out of the box. +Some of the leading open source GUI technologies offering this are +[Jan](https://jan.ai/docs/mcp), [OpenHands](https://github.com/All-Hands-AI/OpenHands/), +[OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp) and [Agno](https://docs.agno.com/introduction/playground). +They allow combining Serena with almost any LLM (including locally running ones) and offer various other integrations. + +### Custom Integrations Agno is a model-agnostic agent framework that allows you to turn Serena into an agent -(independent of the MCP technology) with a large number of underlying LLMs. Agno is currently -the simplest way of running Serena in a chat GUI with an LLM of your choice. - -While Agno is not yet entirely stable, we chose it, because it comes with its own open-source UI, -making it easy to directly use the agent using a chat interface. With Agno, Serena is turned into an agent -(so no longer an MCP Server), so it can be used in programmatic ways (for example for benchmarking or within -your application). +(independent of the MCP technology) with a large number of underlying LLMs. While Agno has recently +added support for MCP servers out of the box, our Agno integration predates this and is a good illustration of how +easy it is to integrate Serena into an arbitrary agent framework. Here's how it works (see also [Agno's documentation](https://docs.agno.com/introduction/playground)): @@ -499,7 +543,7 @@ this in our testing with Claude, allowing this may not be entirely safe. You may choose to disable certain tools for your setup in your Serena project's configuration file (`.yml`). -### Other Agent Frameworks +#### Other Agent Frameworks It should be straightforward to incorporate Serena into any agent framework (like [pydantic-ai](https://ai.pydantic.dev/), [langgraph](https://langchain-ai.github.io/langgraph/tutorials/introduction/) or others). From dfe7a67c68907e8bed86f28a6a819dd2a5c05ef9 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 14:51:48 +0200 Subject: [PATCH 037/220] Changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfbf52c4..9d1152a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ Status of the `main` branch. Changes prior to the next official version change will appear here. +* General: + * **Initial instructions no longer need to be loaded by the user** + * Significantly extended CLI + +* Configuration: + * Simplify customization of modes and contexts, including CLI support. + * Possibility to customize the system prompt and outputs of simple tools, including CLI support. + * Possibility to override tool descriptions through the context YAML. + * Prompt templates are now automatically adapted to the enabled tools. + * Several tools are now excluded by default, need to be included explicitly. + * New context for ChatGPT + * Language servers: * Reliably detect language server termination and propagate the respective error all the way back to the tool application, where an unexpected termination is handled by restarting the language server From a5db2f46162896b37d87a6a9b619fbc96a119153 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 15:53:43 +0200 Subject: [PATCH 038/220] Permissions: set sane mode (755) instead of just executable for downloaded binaries --- src/solidlsp/language_servers/clangd_language_server.py | 3 +-- src/solidlsp/language_servers/clojure_lsp.py | 3 +-- src/solidlsp/language_servers/csharp_language_server.py | 5 ++--- src/solidlsp/language_servers/dart_language_server.py | 3 +-- src/solidlsp/language_servers/eclipse_jdtls.py | 3 +-- src/solidlsp/language_servers/omnisharp.py | 3 +-- src/solidlsp/language_servers/rust_analyzer.py | 3 +-- src/solidlsp/language_servers/terraform_ls.py | 3 +-- 8 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/solidlsp/language_servers/clangd_language_server.py b/src/solidlsp/language_servers/clangd_language_server.py index a770d78e..3879fae7 100644 --- a/src/solidlsp/language_servers/clangd_language_server.py +++ b/src/solidlsp/language_servers/clangd_language_server.py @@ -5,7 +5,6 @@ Provides C/C++ specific instantiation of the LanguageServer class. Contains vari import logging import os import pathlib -import stat import threading from solidlsp.ls import SolidLanguageServer @@ -95,7 +94,7 @@ class ClangdLanguageServer(SolidLanguageServer): f"Clangd executable not found at {clangd_executable_path}.\n" "Make sure you have installed clangd. See https://clangd.llvm.org/installation" ) - os.chmod(clangd_executable_path, stat.S_IEXEC) + os.chmod(clangd_executable_path, 0o755) return clangd_executable_path diff --git a/src/solidlsp/language_servers/clojure_lsp.py b/src/solidlsp/language_servers/clojure_lsp.py index f973db15..d81fa236 100644 --- a/src/solidlsp/language_servers/clojure_lsp.py +++ b/src/solidlsp/language_servers/clojure_lsp.py @@ -6,7 +6,6 @@ import logging import os import pathlib import shutil -import stat import subprocess import threading @@ -118,7 +117,7 @@ class ClojureLSP(SolidLanguageServer): deps.install(logger, clojurelsp_ls_dir) if not os.path.exists(clojurelsp_executable_path): raise FileNotFoundError(f"Download failed? Could not find clojure-lsp executable at {clojurelsp_executable_path}") - os.chmod(clojurelsp_executable_path, stat.S_IEXEC) + os.chmod(clojurelsp_executable_path, 0o755) return clojurelsp_executable_path @staticmethod diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index 66f8c3cb..f01d5b04 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -7,7 +7,6 @@ import logging import os import platform import shutil -import stat import subprocess import tarfile import threading @@ -314,7 +313,7 @@ class CSharpLanguageServer(SolidLanguageServer): # Make executable on Unix systems if platform.system().lower() != "windows": - server_dll.chmod(server_dll.stat().st_mode | stat.S_IEXEC) + server_dll.chmod(0o755) logger.log(f"Successfully installed Microsoft.CodeAnalysis.LanguageServer to {server_dll}", logging.INFO) return str(server_dll) @@ -456,7 +455,7 @@ class CSharpLanguageServer(SolidLanguageServer): # Make dotnet executable on Unix if platform.system().lower() != "windows": - dotnet_exe.chmod(dotnet_exe.stat().st_mode | stat.S_IEXEC) + dotnet_exe.chmod(0o755) logger.log(f"Successfully installed .NET 9 runtime to {dotnet_exe}", logging.INFO) return str(dotnet_exe) diff --git a/src/solidlsp/language_servers/dart_language_server.py b/src/solidlsp/language_servers/dart_language_server.py index c9cfcad1..297294a4 100644 --- a/src/solidlsp/language_servers/dart_language_server.py +++ b/src/solidlsp/language_servers/dart_language_server.py @@ -1,7 +1,6 @@ import logging import os import pathlib -import stat from solidlsp.ls import SolidLanguageServer from solidlsp.ls_logger import LanguageServerLogger @@ -84,7 +83,7 @@ class DartLanguageServer(SolidLanguageServer): deps.install(logger, dart_ls_dir) assert os.path.exists(dart_executable_path) - os.chmod(dart_executable_path, stat.S_IEXEC) + os.chmod(dart_executable_path, 0o755) return f"{dart_executable_path} language-server --client-id multilspy.dart --client-version 1.2" diff --git a/src/solidlsp/language_servers/eclipse_jdtls.py b/src/solidlsp/language_servers/eclipse_jdtls.py index bf87aab7..d664fb7a 100644 --- a/src/solidlsp/language_servers/eclipse_jdtls.py +++ b/src/solidlsp/language_servers/eclipse_jdtls.py @@ -7,7 +7,6 @@ import logging import os import pathlib import shutil -import stat import threading import uuid from pathlib import PurePath @@ -279,7 +278,7 @@ class EclipseJDTLS(SolidLanguageServer): ): FileUtils.download_and_extract_archive(logger, dependency["url"], vscode_java_path, dependency["archiveType"]) - os.chmod(jre_path, stat.S_IEXEC) + os.chmod(jre_path, 0o755) assert os.path.exists(vscode_java_path) assert os.path.exists(jre_home_path) diff --git a/src/solidlsp/language_servers/omnisharp.py b/src/solidlsp/language_servers/omnisharp.py index c9755fa8..e43287c5 100644 --- a/src/solidlsp/language_servers/omnisharp.py +++ b/src/solidlsp/language_servers/omnisharp.py @@ -6,7 +6,6 @@ import json import logging import os import pathlib -import stat import threading from collections.abc import Iterable @@ -194,7 +193,7 @@ class OmniSharp(SolidLanguageServer): FileUtils.download_and_extract_archive(logger, runtime_dependencies["OmniSharp"]["url"], omnisharp_ls_dir, "zip") omnisharp_executable_path = os.path.join(omnisharp_ls_dir, runtime_dependencies["OmniSharp"]["binaryName"]) assert os.path.exists(omnisharp_executable_path) - os.chmod(omnisharp_executable_path, stat.S_IEXEC) + os.chmod(omnisharp_executable_path, 0o755) razor_omnisharp_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "RazorOmnisharp") if not os.path.exists(razor_omnisharp_ls_dir): diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index 48bae28a..308b653f 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -5,7 +5,6 @@ Provides Rust specific instantiation of the LanguageServer class. Contains vario import logging import os import pathlib -import stat import threading from overrides import override @@ -104,7 +103,7 @@ class RustAnalyzer(SolidLanguageServer): os.makedirs(rustanalyzer_ls_dir) deps.install(logger, rustanalyzer_ls_dir) assert os.path.exists(rustanalyzer_executable_path) - os.chmod(rustanalyzer_executable_path, stat.S_IEXEC) + os.chmod(rustanalyzer_executable_path, 0o755) return rustanalyzer_executable_path diff --git a/src/solidlsp/language_servers/terraform_ls.py b/src/solidlsp/language_servers/terraform_ls.py index e40d9af1..6a374df6 100644 --- a/src/solidlsp/language_servers/terraform_ls.py +++ b/src/solidlsp/language_servers/terraform_ls.py @@ -1,7 +1,6 @@ import logging import os import shutil -import stat import threading from overrides import override @@ -112,7 +111,7 @@ class TerraformLS(SolidLanguageServer): # Make the executable file executable on Unix-like systems if platform_id.value != "win-x64": - os.chmod(terraform_ls_executable_path, stat.S_IEXEC | stat.S_IREAD) + os.chmod(terraform_ls_executable_path, 0o755) return terraform_ls_executable_path From b50d8215864afd8f782a4bc8785d059ceb45649a Mon Sep 17 00:00:00 2001 From: KeisukeKaji <113495285+Keisuke05410@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:07:35 +0900 Subject: [PATCH 039/220] Add PATH lookup for solargraph binary before falling back to 'gem exec' --- src/solidlsp/language_servers/solargraph.py | 29 +++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index a4f7e694..ca148f14 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,6 +7,7 @@ import json import logging import os import pathlib +import shutil import subprocess import threading @@ -56,17 +57,6 @@ class Solargraph(SolidLanguageServer): """ Setup runtime dependencies for Solargraph and return the command to start the server. """ - runtime_dependencies = [ - { - "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", - "installCommand": "gem install solargraph -v 0.51.1", - "binaryName": "solargraph", - "archiveType": "gem", - } - ] - - dependency = runtime_dependencies[0] - # Check if Ruby is installed try: result = subprocess.run(["ruby", "--version"], check=True, capture_output=True, cwd=repository_root_path) @@ -78,6 +68,23 @@ class Solargraph(SolidLanguageServer): raise RuntimeError("Ruby is not installed. Please install Ruby before continuing.") from e # Check if solargraph is installed + # First, try to find solargraph in PATH (includes asdf shims) + solargraph_path = shutil.which("solargraph") + if solargraph_path: + logger.log(f"Found solargraph at: {solargraph_path}", logging.INFO) + return solargraph_path + + # Fallback to gem exec + runtime_dependencies = [ + { + "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", + "installCommand": "gem install solargraph -v 0.51.1", + "binaryName": "solargraph", + "archiveType": "gem", + } + ] + + dependency = runtime_dependencies[0] try: result = subprocess.run( ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path From 1d22a377978812bb71fb347731fd60a2ed962525 Mon Sep 17 00:00:00 2001 From: kght6123 Date: Tue, 5 Aug 2025 00:20:36 +0900 Subject: [PATCH 040/220] Docker: improve documentation for quick setup (#394) --- DOCKER.md | 20 ++++++++++++++++++++ compose.yaml | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/DOCKER.md b/DOCKER.md index 5ee8829c..e2647503 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -68,6 +68,8 @@ SERENA_DASHBOARD_PORT=8080 docker-compose up serena docker-compose up serena-dev ``` +Note: Edit the `compose.yaml` file to customize volume mounts for your projects. + ### Using Docker directly ```bash @@ -83,6 +85,24 @@ docker run -it --rm \ serena ``` +### Using Docker Compose with Merge Compose files + +To use Docker Compose with merge files, you can create a `compose.override.yml` file to customize the configuration: + +```yaml +services: + serena: + # To work with projects, you must mount them as volumes: + volumes: + - ./my-project:/workspace/my-project + - /path/to/another/project:/workspace/another-project + # Add the context for the IDE assistant option: + command: + - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" +``` + +See the [Docker Merge Compose files documentation](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) for more details on using merge files. + ## Accessing the Dashboard Once running, access the web dashboard at: diff --git a/compose.yaml b/compose.yaml index bb8f683e..48923e27 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,10 @@ services: serena: image: serena:latest + # To work with projects, you must mount them as volumes: + # volumes: + # - ./my-project:/workspace/my-project + # - /path/to/another/project:/workspace/another-project build: context: ./ dockerfile: Dockerfile @@ -12,6 +16,8 @@ services: - SERENA_DOCKER=1 command: - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0" + # Add the context for the IDE assistant + # - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" serena-dev: image: serena:dev From d56665d35919488f1ddbacef08981071b13fe90b Mon Sep 17 00:00:00 2001 From: liebertar Date: Tue, 5 Aug 2025 16:20:18 +0900 Subject: [PATCH 041/220] Auto-close dashboard tabs when server stops responding This addresses the dashboard tab accumulation issue that's been brought up. Having multiple tabs open gets disruptive during development. Implemented the approach @opcode81 suggested - detecting when the server stops responding and letting the tab close itself. The dashboard now monitors its connection and auto-closes after 3 failed polling attempts. Should help with the workflow disruption without being overly complex. Hope this helps\! Fixes #388 --- src/serena/resources/dashboard/dashboard.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index 318d0c16..4edc0868 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -52,6 +52,7 @@ class Dashboard { this.toolNames = []; this.currentMaxIdx = -1; this.pollInterval = null; + this.failureCount = 0; this.$logContainer = $('#log-container'); this.$errorContainer = $('#error-container'); this.$loadButton = $('#load-logs'); @@ -161,6 +162,7 @@ class Dashboard { start_idx: self.currentMaxIdx + 1 }), success: function(response) { + self.failureCount = 0; // Only append new messages if we have any if (response.messages && response.messages.length > 0) { let wasAtBottom = false; @@ -190,6 +192,11 @@ class Dashboard { }, error: function(xhr, status, error) { console.error('Error polling for new logs:', error); + self.failureCount++; + if (self.failureCount >= 3) { + console.log('Server appears to be down, closing tab'); + window.close(); + } } }); } From b931e972434a7c50ddf4edf92fee77e5db8d8053 Mon Sep 17 00:00:00 2001 From: Kai Yao Date: Tue, 5 Aug 2025 11:15:35 -0500 Subject: [PATCH 042/220] added support for rust 2024 edition (#412) --- .../language_servers/rust_analyzer.py | 10 +- .../repos/rust/test_repo_2024/Cargo.lock | 7 ++ .../repos/rust/test_repo_2024/Cargo.toml | 6 + .../repos/rust/test_repo_2024/src/lib.rs | 21 ++++ .../repos/rust/test_repo_2024/src/main.rs | 19 +++ test/solidlsp/rust/test_rust_2024_edition.py | 112 ++++++++++++++++++ 6 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 test/resources/repos/rust/test_repo_2024/Cargo.lock create mode 100644 test/resources/repos/rust/test_repo_2024/Cargo.toml create mode 100644 test/resources/repos/rust/test_repo_2024/src/lib.rs create mode 100644 test/resources/repos/rust/test_repo_2024/src/main.rs create mode 100644 test/solidlsp/rust/test_rust_2024_edition.py diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index de73f045..d9b09090 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -60,7 +60,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-apple-darwin.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-apple-darwin.gz", platform_id="osx-x64", archive_type="gz", binary_name="rust_analyzer", @@ -68,7 +68,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (arm64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-aarch64-apple-darwin.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-aarch64-apple-darwin.gz", platform_id="osx-arm64", archive_type="gz", binary_name="rust_analyzer", @@ -76,7 +76,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Linux (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-unknown-linux-gnu.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-unknown-linux-gnu.gz", platform_id="linux-x64", archive_type="gz", binary_name="rust_analyzer", @@ -84,7 +84,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Linux (arm64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-aarch64-unknown-linux-gnu.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-aarch64-unknown-linux-gnu.gz", platform_id="linux-arm64", archive_type="gz", binary_name="rust_analyzer", @@ -92,7 +92,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Windows (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-pc-windows-msvc.zip", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-pc-windows-msvc.zip", platform_id="win-x64", archive_type="zip", binary_name="rust-analyzer.exe", diff --git a/test/resources/repos/rust/test_repo_2024/Cargo.lock b/test/resources/repos/rust/test_repo_2024/Cargo.lock new file mode 100644 index 00000000..88beb720 --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "rsandbox_2024" +version = "0.1.0" diff --git a/test/resources/repos/rust/test_repo_2024/Cargo.toml b/test/resources/repos/rust/test_repo_2024/Cargo.toml new file mode 100644 index 00000000..2f20bf2e --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "rsandbox_2024" +version = "0.1.0" +edition = "2024" + +[dependencies] \ No newline at end of file diff --git a/test/resources/repos/rust/test_repo_2024/src/lib.rs b/test/resources/repos/rust/test_repo_2024/src/lib.rs new file mode 100644 index 00000000..28fac4c7 --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/src/lib.rs @@ -0,0 +1,21 @@ +pub fn multiply(a: i32, b: i32) -> i32 { + a * b +} + +pub struct Calculator { + pub result: i32, +} + +impl Calculator { + pub fn new() -> Self { + Calculator { result: 0 } + } + + pub fn add(&mut self, value: i32) { + self.result += value; + } + + pub fn get_result(&self) -> i32 { + self.result + } +} \ No newline at end of file diff --git a/test/resources/repos/rust/test_repo_2024/src/main.rs b/test/resources/repos/rust/test_repo_2024/src/main.rs new file mode 100644 index 00000000..ec20c47a --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/src/main.rs @@ -0,0 +1,19 @@ +fn main() { + println!("Hello, Rust 2024 edition!"); + let result = add(2, 3); + println!("2 + 3 = {}", result); +} + +pub fn add(a: i32, b: i32) -> i32 { + a + b +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + assert_eq!(add(2, 3), 5); + } +} \ No newline at end of file diff --git a/test/solidlsp/rust/test_rust_2024_edition.py b/test/solidlsp/rust/test_rust_2024_edition.py new file mode 100644 index 00000000..e73c8d6d --- /dev/null +++ b/test/solidlsp/rust/test_rust_2024_edition.py @@ -0,0 +1,112 @@ +import os +from pathlib import Path + +import pytest + +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils +from test.conftest import create_ls + + +@pytest.mark.rust +class TestRust2024EditionLanguageServer: + @classmethod + def setup_class(cls): + """Set up the test class with the Rust 2024 edition test repository.""" + cls.test_repo_2024_path = Path(__file__).parent.parent.parent / "resources" / "repos" / "rust" / "test_repo_2024" + + if not cls.test_repo_2024_path.exists(): + pytest.skip("Rust 2024 edition test repository not found") + + # Create and start the language server for the 2024 edition repo + cls.language_server = create_ls(Language.RUST, str(cls.test_repo_2024_path)) + cls.language_server.start() + + @classmethod + def teardown_class(cls): + """Clean up the language server.""" + if hasattr(cls, "language_server"): + cls.language_server.stop() + + def test_find_references_raw(self) -> None: + # Test finding references to the 'add' function defined in main.rs + file_path = os.path.join("src", "main.rs") + symbols = self.language_server.request_document_symbols(file_path) + add_symbol = None + for sym in symbols[0]: + if sym.get("name") == "add": + add_symbol = sym + break + assert add_symbol is not None, "Could not find 'add' function symbol in main.rs" + sel_start = add_symbol["selectionRange"]["start"] + refs = self.language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + # The add function should be referenced within main.rs itself (in the main function) + assert any("main.rs" in ref.get("relativePath", "") for ref in refs), "main.rs should reference add function" + + def test_find_symbol(self) -> None: + symbols = self.language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "multiply"), "multiply function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator struct not found in symbol tree" + + def test_find_referencing_symbols_multiply(self) -> None: + # Find references to 'multiply' function defined in lib.rs + file_path = os.path.join("src", "lib.rs") + symbols = self.language_server.request_document_symbols(file_path) + multiply_symbol = None + for sym in symbols[0]: + if sym.get("name") == "multiply": + multiply_symbol = sym + break + assert multiply_symbol is not None, "Could not find 'multiply' function symbol in lib.rs" + sel_start = multiply_symbol["selectionRange"]["start"] + refs = self.language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + # The multiply function exists but may not be referenced anywhere, which is fine + # This test just verifies we can find the symbol and request references without error + assert isinstance(refs, list), "Should return a list of references (even if empty)" + + def test_find_calculator_struct_and_impl(self) -> None: + # Test finding the Calculator struct and its impl block + file_path = os.path.join("src", "lib.rs") + symbols = self.language_server.request_document_symbols(file_path) + + # Find the Calculator struct + calculator_struct = None + calculator_impl = None + for sym in symbols[0]: + if sym.get("name") == "Calculator" and sym.get("kind") == 23: # Struct kind + calculator_struct = sym + elif sym.get("name") == "Calculator" and sym.get("kind") == 11: # Interface/Impl kind + calculator_impl = sym + + assert calculator_struct is not None, "Could not find 'Calculator' struct symbol in lib.rs" + + # The struct should have the 'result' field + struct_children = calculator_struct.get("children", []) + field_names = [child.get("name") for child in struct_children] + assert "result" in field_names, "Calculator struct should have 'result' field" + + # Find the impl block and check its methods + if calculator_impl is not None: + impl_children = calculator_impl.get("children", []) + method_names = [child.get("name") for child in impl_children] + assert "new" in method_names, "Calculator impl should have 'new' method" + assert "add" in method_names, "Calculator impl should have 'add' method" + assert "get_result" in method_names, "Calculator impl should have 'get_result' method" + + def test_overview_methods(self) -> None: + symbols = self.language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "multiply"), "multiply missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator missing from overview" + + def test_rust_2024_edition_specific(self) -> None: + # Verify we're actually working with the 2024 edition repository + cargo_toml_path = self.test_repo_2024_path / "Cargo.toml" + assert cargo_toml_path.exists(), "Cargo.toml should exist in test repository" + + with open(cargo_toml_path) as f: + content = f.read() + assert 'edition = "2024"' in content, "Should be using Rust 2024 edition" From 396ca3f356a2fea7a429ba6ed675bb4e650fb1d1 Mon Sep 17 00:00:00 2001 From: VH <80348187+vhbui02@users.noreply.github.com> Date: Wed, 6 Aug 2025 11:34:01 +0000 Subject: [PATCH 043/220] Add NodeJS inside Docker image (#421) --- Dockerfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d88d4e26..205f4ee6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ FROM python:3.11-slim AS base SHELL ["/bin/bash", "-c"] # Set environment variables to make Python print directly to the terminal and avoid .pyc files. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 # Install system dependencies required for package manager and build tools. RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -18,6 +18,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN python3 -m pip install --no-cache-dir pipx \ && pipx ensurepath +# Install nodejs +ENV NVM_VERSION=0.40.3 +ENV NODE_VERSION=22.18.0 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash +# standard location +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}" + # Add local bin to the path ENV PATH="${PATH}:/root/.local/bin" From 7158aa4870f0c1e08117322c0eff72b80d4fd9bb Mon Sep 17 00:00:00 2001 From: hyuraku <32809703+hyuraku@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:21:43 +0900 Subject: [PATCH 044/220] Adds tests for Ruby language server integration (#416) --- .github/workflows/pytest.yml | 7 ++++ pyproject.toml | 1 + .../repos/ruby/test_repo/.solargraph.yml | 4 ++ test/resources/repos/ruby/test_repo/lib.rb | 9 ++++ test/resources/repos/ruby/test_repo/main.rb | 21 ++++++++++ test/solidlsp/ruby/test_ruby_basic.py | 42 +++++++++++++++++++ 6 files changed, 84 insertions(+) create mode 100644 test/resources/repos/ruby/test_repo/.solargraph.yml create mode 100644 test/resources/repos/ruby/test_repo/lib.rb create mode 100644 test/resources/repos/ruby/test_repo/main.rb create mode 100644 test/solidlsp/ruby/test_ruby_basic.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3eeac6be..ddfe7a32 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -69,6 +69,13 @@ jobs: with: terraform_version: "1.5.0" terraform_wrapper: false + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + - name: Install Ruby language server + shell: bash + run: gem install solargraph - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/pyproject.toml b/pyproject.toml index 8872384c..d46c678f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -260,6 +260,7 @@ markers = [ "elixir: language server running for Elixir", "terraform: language server running for Terraform", "snapshot: snapshot tests for symbolic editing operations", + "ruby: language server running for Ruby", ] [tool.codespell] diff --git a/test/resources/repos/ruby/test_repo/.solargraph.yml b/test/resources/repos/ruby/test_repo/.solargraph.yml new file mode 100644 index 00000000..b4678eec --- /dev/null +++ b/test/resources/repos/ruby/test_repo/.solargraph.yml @@ -0,0 +1,4 @@ +--- +include: + - "main.rb" + - "lib.rb" diff --git a/test/resources/repos/ruby/test_repo/lib.rb b/test/resources/repos/ruby/test_repo/lib.rb new file mode 100644 index 00000000..5dd8504b --- /dev/null +++ b/test/resources/repos/ruby/test_repo/lib.rb @@ -0,0 +1,9 @@ +class Calculator + def add(a, b) + a + b + end + + def subtract(a, b) + a - b + end +end diff --git a/test/resources/repos/ruby/test_repo/main.rb b/test/resources/repos/ruby/test_repo/main.rb new file mode 100644 index 00000000..dcf1b0f2 --- /dev/null +++ b/test/resources/repos/ruby/test_repo/main.rb @@ -0,0 +1,21 @@ +require './lib.rb' + +class DemoClass + attr_accessor :value + + def initialize(value) + @value = value + end + + def print_value + puts @value + end +end + +def helper_function(number = 42) + demo = DemoClass.new(number) + Calculator.new.add(demo.value, 10) + demo.print_value +end + +helper_function diff --git a/test/solidlsp/ruby/test_ruby_basic.py b/test/solidlsp/ruby/test_ruby_basic.py new file mode 100644 index 00000000..435c0538 --- /dev/null +++ b/test/solidlsp/ruby/test_ruby_basic.py @@ -0,0 +1,42 @@ +import os +from pathlib import Path + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils + + +@pytest.mark.ruby +class TestRubyLanguageServer: + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + def test_find_symbol(self, language_server: SolidLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "DemoClass"), "DemoClass not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "helper_function"), "helper_function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "print_value"), "print_value not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: + file_path = os.path.join("main.rb") + symbols = language_server.request_document_symbols(file_path) + helper_symbol = None + for sym in symbols[0]: + if sym.get("name") == "helper_function": + helper_symbol = sym + break + print(helper_symbol) + assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" + + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) + def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: + # Test finding Calculator.add method definition from line 17: Calculator.new.add(demo.value, 10) + definition_location_list = language_server.request_definition(str(repo_path / "main.rb"), 16,17) # add method at line 17 (0-indexed 16), position 17 + + assert len(definition_location_list) == 1 + definition_location = definition_location_list[0] + print(f"Found definition: {definition_location}") + assert definition_location["uri"].endswith("lib.rb") + assert definition_location["range"]["start"]["line"] == 1 # add method on line 2 (0-indexed 1) From ec1277167b7ec26c833bc4decd5df7c7ec007da0 Mon Sep 17 00:00:00 2001 From: Yen <5915590+antigenius0910@users.noreply.github.com> Date: Wed, 6 Aug 2025 09:41:04 -0500 Subject: [PATCH 045/220] Add comprehensive bash language support (#375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add comprehensive bash language support Implement BashLanguageServer using bash-language-server (v5.4.0) - Add automatic Node.js/npm dependency management - Support .sh and .bash file extensions - Include cross-platform compatibility (Windows .cmd handling) - Add comprehensive test infrastructure with realistic bash scripts - Integrate bash into Language enum and language server factory - Add bash pytest marker for selective testing - Include bash_language_server fixture for tests Features: * LSP capabilities: text sync, completion, hover, document symbols * Runtime dependency installation via npm * Symbol recognition for functions, variables, conditionals * Proper LSP initialization and error handling * Test repository with main.sh, utils.sh, and config.sh examples Files: + src/solidlsp/language_servers/bash_language_server.py (new implementation) + test/solidlsp/bash/test_bash_basic.py (comprehensive test suite) + test/resources/repos/bash/test_repo/ (test scripts) * src/solidlsp/ls_config.py (add BASH enum and file patterns) * src/solidlsp/ls.py (integrate bash language server) * pyproject.toml (add bash pytest marker) * test/conftest.py (add bash_language_server fixture) Tests: 2 passed, 7 appropriately skipped All existing tests continue to pass (288 passed total) * trigger flow * feat: enhance bash function detection with hybrid LSP + regex approach - Upgrade bash-language-server from 5.4.0 to 5.6.0 for better capabilities - Add hybrid function detection: primary LSP + regex fallback when needed - Implement _detect_bash_functions() with regex patterns for both syntax types: * function name() { } (function keyword syntax) * name() { } (regular function syntax) - Add _find_function_end() with brace matching for accurate function boundaries - Enhance request_document_symbols() with intelligent fallback logic - Add comprehensive test suite with 4 new test functions: * test_bash_function_detection_hybrid_approach * test_bash_function_detection_with_body * test_bash_regex_fallback_detection * test_bash_function_syntax_patterns - All tests passing: 6 passed, 7 skipped, 0 failed - Maintains 100% backward compatibility - Enables full symbolic operations for bash functions: find_symbol, replace_symbol_body, etc. * remove trash * clean: remove debug changes from Python services.py - Remove 'MODIFIED VIA SYMBOLIC OPERATIONS!' debug comments - Remove debug print statements added during testing - Restore original clean docstring and code - No functional changes to bash language support implementation * fix: Remove non-bash specific directory ignoring from BashLanguageServer - Removes is_ignored_dirname override that included generic directories like node_modules, .cache, tmp, temp - These directories aren't bash-specific and gitignore already handles file exclusion - Addresses PR feedback: 'that doesn't seem bash related. Note that gitignored files are ignored anyway' * fix: Restructure bash tests to follow standard patterns and remove pytest.skip - Replace custom bash_language_server fixture with standard parametrized language_server fixture - Remove all pytest.skip statements and replace with proper assertions or failures - Remove tests for non-existent methods (get_symbols_for_document, get_workspace_symbols, etc.) - Keep only tests that use actual SolidLanguageServer methods like request_document_symbols - All bash tests now pass successfully - Addresses PR feedback about test structure and pytest.skip removal * docs: Add bash language support to README - Add Bash to the list of directly supported languages - Note that it requires Node.js and npm for bash-language-server installation - Addresses PR feedback about missing README extension for bash support * docs: Add bash language support to CHANGELOG - Document comprehensive bash language support with bash-language-server integration - Highlight key features: .sh/.bash extensions, hybrid function detection, automatic dependency management - Note cross-platform compatibility including Windows support - Addresses PR feedback about missing CHANGELOG extension for bash support * refactor: Change regex fallback to always-on hybrid detection for bash functions - Remove conditional logic that only ran regex when LSP found no functions - Implement always-on hybrid approach that runs both LSP and regex detection - Add comprehensive documentation explaining bash-language-server limitations - Add new _merge_function_detections method to combine results without duplicates - Addresses reviewer feedback about files with mixed function notation styles 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --------- Co-authored-by: Claude --- CHANGELOG.md | 5 + README.md | 1 + pyproject.toml | 1 + .../language_servers/bash_language_server.py | 388 ++++++++++++++++++ src/solidlsp/ls.py | 5 + src/solidlsp/ls_config.py | 3 + test/conftest.py | 3 + test/resources/repos/bash/test_repo/config.sh | 91 ++++ test/resources/repos/bash/test_repo/main.sh | 70 ++++ test/resources/repos/bash/test_repo/utils.sh | 83 ++++ test/solidlsp/bash/__init__.py | 0 test/solidlsp/bash/test_bash_basic.py | 137 +++++++ 12 files changed, 787 insertions(+) create mode 100644 src/solidlsp/language_servers/bash_language_server.py create mode 100644 test/resources/repos/bash/test_repo/config.sh create mode 100644 test/resources/repos/bash/test_repo/main.sh create mode 100644 test/resources/repos/bash/test_repo/utils.sh create mode 100644 test/solidlsp/bash/__init__.py create mode 100644 test/solidlsp/bash/test_bash_basic.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dfbf52c4..a32f46cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Status of the `main` branch. Changes prior to the next official version change w * Reliably detect language server termination and propagate the respective error all the way back to the tool application, where an unexpected termination is handled by restarting the language server and subsequently retrying the tool application. + * Add comprehensive bash language support with bash-language-server integration + * Support for .sh and .bash file extensions + * Hybrid function detection (LSP + regex fallback) for reliable symbolic editing + * Automatic Node.js/npm dependency management + * Cross-platform compatibility including Windows Fixes: * Ignore `.git` in check for ignored paths and improve performance of `find_all_non_ignored_files` diff --git a/README.md b/README.md index 1b6777cb..1d8d1d3d 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ With Serena, we provide * Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.) * Elixir (Requires NextLS and Elixir install; **Windows not supported** - Next LS does not provide Windows binaries) * Clojure + * Bash (Requires Node.js and npm for bash-language-server installation) * C/C++ (You may experience issues with finding references, we are working on it) * indirect support (may require some code changes/manual installation) for: * Ruby (untested) diff --git a/pyproject.toml b/pyproject.toml index d46c678f..8e4d563c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -259,6 +259,7 @@ markers = [ "csharp: language server running for C#", "elixir: language server running for Elixir", "terraform: language server running for Terraform", + "bash: language server running for Bash", "snapshot: snapshot tests for symbolic editing operations", "ruby: language server running for Ruby", ] diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py new file mode 100644 index 00000000..04ffe355 --- /dev/null +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -0,0 +1,388 @@ +""" +Provides Bash specific instantiation of the LanguageServer class using bash-language-server. +Contains various configurations and settings specific to Bash scripting. +""" + +import logging +import os +import pathlib +import shutil +import threading + +from overrides import override + +from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + + +from solidlsp import ls_types + +class BashLanguageServer(SolidLanguageServer): + """ + Provides Bash specific instantiation of the LanguageServer class using bash-language-server. + Contains various configurations and settings specific to Bash scripting. + """ + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates a BashLanguageServer instance. This class is not meant to be instantiated directly. + Use LanguageServer.create() instead. + """ + bash_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings) + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=bash_lsp_executable_path, cwd=repository_root_path), + "bash", + solidlsp_settings, + ) + self.server_ready = threading.Event() + self.initialize_searcher_command_available = threading.Event() + + + + @classmethod + def _setup_runtime_dependencies( + cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings + ) -> str: + """ + Setup runtime dependencies for Bash Language Server and return the command to start the server. + """ + # Verify both node and npm are installed + is_node_installed = shutil.which("node") is not None + assert is_node_installed, "node is not installed or isn't in PATH. Please install NodeJS and try again." + is_npm_installed = shutil.which("npm") is not None + assert is_npm_installed, "npm is not installed or isn't in PATH. Please install npm and try again." + + deps = RuntimeDependencyCollection( + [ + RuntimeDependency( + id="bash-language-server", + description="bash-language-server package", + command="npm install --prefix ./ bash-language-server@5.6.0", + platform_id="any", + ), + ] + ) + + # Install bash-language-server if not already installed + bash_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "bash-lsp") + bash_executable_path = os.path.join(bash_ls_dir, "node_modules", ".bin", "bash-language-server") + + # Handle Windows executable extension + if os.name == "nt": + bash_executable_path += ".cmd" + + if not os.path.exists(bash_executable_path): + logger.log(f"Bash Language Server executable not found at {bash_executable_path}. Installing...", logging.INFO) + deps.install(logger, bash_ls_dir) + logger.log("Bash language server dependencies installed successfully", logging.INFO) + + if not os.path.exists(bash_executable_path): + raise FileNotFoundError( + f"bash-language-server executable not found at {bash_executable_path}, something went wrong with the installation." + ) + return f"{bash_executable_path} start" + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Bash Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + initialize_params = { + "locale": "en", + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, + "signatureHelp": {"dynamicRegistration": True}, + "codeAction": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "processId": os.getpid(), + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + return initialize_params + + def _start_server(self): + """ + Starts the Bash Language Server, waits for the server to be ready and yields the LanguageServer instance. + """ + + def register_capability_handler(params): + assert "registrations" in params + for registration in params["registrations"]: + if registration["method"] == "workspace/executeCommand": + self.initialize_searcher_command_available.set() + return + + def execute_client_command_handler(params): + return [] + + def do_nothing(params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + # Check for bash-language-server ready signals + message_text = msg.get("message", "") + if "Analyzing" in message_text or "analysis complete" in message_text.lower(): + self.logger.log("Bash language server analysis signals detected", logging.INFO) + self.server_ready.set() + self.completions_available.set() + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_request("workspace/executeClientCommand", execute_client_command_handler) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting Bash server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + self.logger.log(f"Received initialize response from bash server: {init_response}", logging.DEBUG) + + # Enhanced capability checks for bash-language-server 5.6.0 + assert init_response["capabilities"]["textDocumentSync"] in [1, 2] # Full or Incremental + assert "completionProvider" in init_response["capabilities"] + + # Verify document symbol support is available + if "documentSymbolProvider" in init_response["capabilities"]: + self.logger.log("Bash server supports document symbols", logging.INFO) + else: + self.logger.log("Warning: Bash server does not report document symbol support", logging.WARNING) + + self.server.notify.initialized({}) + + # Wait for server readiness with timeout + self.logger.log("Waiting for Bash language server to be ready...", logging.INFO) + if not self.server_ready.wait(timeout=3.0): + # Fallback: assume server is ready after timeout + self.logger.log("Timeout waiting for bash server ready signal, proceeding anyway", logging.WARNING) + self.server_ready.set() + self.completions_available.set() + else: + self.logger.log("Bash server initialization complete", logging.INFO) + + def request_document_symbols( + self, relative_file_path: str, include_body: bool = False + ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + Enhanced document symbol request with hybrid LSP + regex-based function detection for bash files. + + This method combines both LSP-based detection and regex-based detection to provide comprehensive + function discovery. This dual approach is necessary because: + + 1. bash-language-server (v5.6.0) has inconsistent function detection capabilities + 2. Some bash function syntaxes are not reliably detected by the LSP server + 3. Files may contain mixed function notation styles within the same file + 4. Different formatting or indentation can affect LSP detection + + The hybrid approach ensures maximum compatibility and comprehensive function discovery + for reliable symbolic editing operations in Serena. + """ + # First get symbols from the standard LSP approach + lsp_all_symbols, lsp_root_symbols = super().request_document_symbols(relative_file_path, include_body) + + # Always run regex-based function detection for comprehensive coverage + # This addresses bash-language-server limitations and ensures we catch all function patterns + self.logger.log(f"Running hybrid function detection (LSP + regex) for {relative_file_path}", logging.DEBUG) + + regex_detected_functions = self._detect_bash_functions(relative_file_path, include_body) + + # Merge LSP and regex results, avoiding duplicates + merged_all_symbols, merged_root_symbols = self._merge_function_detections( + lsp_all_symbols, lsp_root_symbols, regex_detected_functions + ) + + # Log detection results for debugging + lsp_functions = [s for s in lsp_all_symbols if s.get("kind") == 12] + total_functions = [s for s in merged_all_symbols if s.get("kind") == 12] + + self.logger.log( + f"Function detection for {relative_file_path}: LSP={len(lsp_functions)}, " + f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", + logging.INFO + ) + + return merged_all_symbols, merged_root_symbols + + def _merge_function_detections( + self, + lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], + lsp_root_symbols: list[ls_types.UnifiedSymbolInformation], + regex_detected_functions: list[ls_types.UnifiedSymbolInformation] + ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + Merge LSP-detected symbols with regex-detected functions, avoiding duplicates. + + This method: + 1. Keeps all non-function symbols from LSP detection + 2. Keeps all LSP-detected functions (they have more accurate positioning) + 3. Adds regex-detected functions that weren't found by LSP + 4. Uses function names to detect duplicates + + Args: + lsp_all_symbols: All symbols detected by LSP + lsp_root_symbols: Root-level symbols detected by LSP + regex_detected_functions: Functions detected by regex (all are root-level) + + Returns: + Tuple of (merged_all_symbols, merged_root_symbols) + """ + # Extract function names that LSP already detected + lsp_function_names = set() + for symbol in lsp_all_symbols: + if symbol.get("kind") == 12: # LSP Symbol Kind 12 = Function + lsp_function_names.add(symbol["name"]) + + # Start with all LSP symbols (both functions and non-functions) + merged_all_symbols = lsp_all_symbols.copy() + merged_root_symbols = lsp_root_symbols.copy() + + # Add regex-detected functions that weren't found by LSP + added_functions = 0 + for regex_function in regex_detected_functions: + function_name = regex_function["name"] + + # Only add if this function wasn't detected by LSP + if function_name not in lsp_function_names: + merged_all_symbols.append(regex_function) + merged_root_symbols.append(regex_function) # All regex functions are root-level + added_functions += 1 + self.logger.log(f"Added regex-detected function '{function_name}' not found by LSP", logging.DEBUG) + + if added_functions > 0: + self.logger.log(f"Merged {added_functions} additional functions from regex detection", logging.INFO) + + return merged_all_symbols, merged_root_symbols + + def _detect_bash_functions(self, relative_file_path: str, include_body: bool = False) -> list[ls_types.UnifiedSymbolInformation]: + """ + Regex-based detection of bash functions as fallback when LSP doesn't provide them. + """ + import re + from solidlsp import ls_types + + try: + # Read the file content directly from filesystem + abs_path = os.path.join(self.repository_root_path, relative_file_path) + with open(abs_path, 'r', encoding='utf-8') as f: + file_content = f.read() + lines = file_content.split("\n") + + detected_functions = [] + + # Regex patterns for bash function definitions + # Pattern 1: function name() { ... } + # Pattern 2: name() { ... } + function_patterns = [ + re.compile(r'^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{'), + re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{') + ] + + for line_num, line in enumerate(lines): + for pattern in function_patterns: + match = pattern.match(line) + if match: + func_name = match.group(1) + + # Find the end of the function by matching braces + start_line = line_num + end_line = self._find_function_end(lines, start_line) + + # Create location information + location = ls_types.Location( + uri=pathlib.Path(abs_path).as_uri(), + range={ + "start": {"line": start_line, "character": match.start(1)}, + "end": {"line": end_line, "character": 0} + }, + absolutePath=abs_path, + relativePath=relative_file_path + ) + + # Create the function symbol + func_symbol = ls_types.UnifiedSymbolInformation( + name=func_name, + kind=12, # LSP Symbol Kind for Function + location=location, + range=location["range"], + selectionRange={ + "start": {"line": start_line, "character": match.start(1)}, + "end": {"line": start_line, "character": match.end(1)} + }, + children=[], + parent=None + ) + + # Add function body if requested + if include_body: + func_symbol["body"] = "\n".join(lines[start_line:end_line + 1]) + + detected_functions.append(func_symbol) + break + + return detected_functions + + except Exception as e: + self.logger.log(f"Error in regex-based function detection for {relative_file_path}: {e}", logging.WARNING) + return [] + + def _find_function_end(self, lines: list[str], start_line: int) -> int: + """ + Find the end line of a bash function by matching opening and closing braces. + """ + brace_count = 0 + in_function = False + + for i in range(start_line, len(lines)): + line = lines[i] + + # Count braces, handling basic cases + for char in line: + if char == '{': + brace_count += 1 + in_function = True + elif char == '}' and in_function: + brace_count -= 1 + if brace_count == 0: + return i + + # Fallback: if we can't match braces properly, assume single line or small function + return min(start_line + 10, len(lines) - 1) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index ded2645a..edda2d6e 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -226,6 +226,11 @@ class SolidLanguageServer(ABC): ls = TerraformLS(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + elif config.code_language == Language.BASH: + from solidlsp.language_servers.bash_language_server import BashLanguageServer + + ls = BashLanguageServer(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + else: logger.log(f"Language {config.code_language} is not supported", logging.ERROR) raise SolidLSPException(f"Language {config.code_language} is not supported") diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index efb55a87..3965d066 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -42,6 +42,7 @@ class Language(str, Enum): CLOJURE = "clojure" ELIXIR = "elixir" TERRAFORM = "terraform" + BASH = "bash" # Experimental or deprecated Language Servers TYPESCRIPT_VTS = "typescript_vts" """Use the typescript language server through the natively bundled vscode extension via https://github.com/yioneko/vtsls""" @@ -103,6 +104,8 @@ class Language(str, Enum): return FilenameMatcher("*.ex", "*.exs") case self.TERRAFORM: return FilenameMatcher("*.tf", "*.tfvars", "*.tfstate") + case self.BASH: + return FilenameMatcher("*.sh", "*.bash") case _: raise ValueError(f"Unhandled language: {self}") diff --git a/test/conftest.py b/test/conftest.py index 849af543..da91ebdd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -144,3 +144,6 @@ def project(request: LanguageParamRequest): language = request.param yield create_default_project(language) + + + diff --git a/test/resources/repos/bash/test_repo/config.sh b/test/resources/repos/bash/test_repo/config.sh new file mode 100644 index 00000000..440867e5 --- /dev/null +++ b/test/resources/repos/bash/test_repo/config.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Configuration script for project setup + +# Environment variables +export PROJECT_NAME="bash-test-project" +export PROJECT_VERSION="1.0.0" +export LOG_LEVEL="INFO" +export CONFIG_DIR="./config" + +# Default settings +DEFAULT_TIMEOUT=30 +DEFAULT_RETRIES=3 +DEFAULT_PORT=8080 + +# Configuration arrays +declare -A ENVIRONMENTS=( + ["dev"]="development" + ["prod"]="production" + ["test"]="testing" +) + +declare -A DATABASE_CONFIGS=( + ["host"]="localhost" + ["port"]="5432" + ["name"]="myapp_db" + ["user"]="dbuser" +) + +# Function to load configuration +load_config() { + local env="${1:-dev}" + local config_file="${CONFIG_DIR}/${env}.conf" + + if [[ -f "$config_file" ]]; then + echo "Loading configuration from $config_file" + source "$config_file" + else + echo "Warning: Configuration file $config_file not found, using defaults" + fi +} + +# Function to validate configuration +validate_config() { + local errors=0 + + if [[ -z "$PROJECT_NAME" ]]; then + echo "Error: PROJECT_NAME is not set" >&2 + ((errors++)) + fi + + if [[ -z "$PROJECT_VERSION" ]]; then + echo "Error: PROJECT_VERSION is not set" >&2 + ((errors++)) + fi + + if [[ $DEFAULT_PORT -lt 1024 || $DEFAULT_PORT -gt 65535 ]]; then + echo "Error: Invalid port number $DEFAULT_PORT" >&2 + ((errors++)) + fi + + return $errors +} + +# Function to print configuration +print_config() { + echo "=== Current Configuration ===" + echo "Project Name: $PROJECT_NAME" + echo "Version: $PROJECT_VERSION" + echo "Log Level: $LOG_LEVEL" + echo "Default Port: $DEFAULT_PORT" + echo "Default Timeout: $DEFAULT_TIMEOUT" + echo "Default Retries: $DEFAULT_RETRIES" + + echo "\n=== Environments ===" + for env in "${!ENVIRONMENTS[@]}"; do + echo " $env: ${ENVIRONMENTS[$env]}" + done + + echo "\n=== Database Configuration ===" + for key in "${!DATABASE_CONFIGS[@]}"; do + echo " $key: ${DATABASE_CONFIGS[$key]}" + done +} + +# Initialize configuration if this script is run directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + load_config "$1" + validate_config + print_config +fi diff --git a/test/resources/repos/bash/test_repo/main.sh b/test/resources/repos/bash/test_repo/main.sh new file mode 100644 index 00000000..56d2e72e --- /dev/null +++ b/test/resources/repos/bash/test_repo/main.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Main script demonstrating various bash features + +# Global variables +readonly SCRIPT_NAME="Main Script" +COUNTER=0 +declare -a ITEMS=("item1" "item2" "item3") + +# Function definitions +function greet_user() { + local username="$1" + local greeting_type="${2:-default}" + + case "$greeting_type" in + "formal") + echo "Good day, ${username}!" + ;; + "casual") + echo "Hey ${username}!" + ;; + *) + echo "Hello, ${username}!" + ;; + esac +} + +function process_items() { + local -n items_ref=$1 + local operation="$2" + + for item in "${items_ref[@]}"; do + case "$operation" in + "count") + ((COUNTER++)) + echo "Processing item $COUNTER: $item" + ;; + "uppercase") + echo "${item^^}" + ;; + *) + echo "Unknown operation: $operation" + return 1 + ;; + esac + done +} + +# Main execution +main() { + echo "Starting $SCRIPT_NAME" + + if [[ $# -eq 0 ]]; then + echo "Usage: $0 [greeting_type]" + exit 1 + fi + + local user="$1" + local greeting="${2:-default}" + + greet_user "$user" "$greeting" + + echo "Processing items..." + process_items ITEMS "count" + + echo "Script completed successfully" +} + +# Run main function with all arguments +main "$@" diff --git a/test/resources/repos/bash/test_repo/utils.sh b/test/resources/repos/bash/test_repo/utils.sh new file mode 100644 index 00000000..77fc8fed --- /dev/null +++ b/test/resources/repos/bash/test_repo/utils.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Utility functions for bash scripting + +# String manipulation functions +function to_uppercase() { + echo "${1^^}" +} + +function to_lowercase() { + echo "${1,,}" +} + +function trim_whitespace() { + local var="$1" + var="${var#"${var%%[![:space:]]*}"}" + var="${var%"${var##*[![:space:]]}"}" + echo "$var" +} + +# File operations +function backup_file() { + local file="$1" + local backup_dir="${2:-./backups}" + + if [[ ! -f "$file" ]]; then + echo "Error: File '$file' does not exist" >&2 + return 1 + fi + + mkdir -p "$backup_dir" + cp "$file" "${backup_dir}/$(basename "$file").$(date +%Y%m%d_%H%M%S).bak" + echo "Backup created for $file" +} + +# Array operations +function contains_element() { + local element="$1" + shift + local array=("$@") + + for item in "${array[@]}"; do + if [[ "$item" == "$element" ]]; then + return 0 + fi + done + return 1 +} + +# Logging functions +function log_message() { + local level="$1" + local message="$2" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + case "$level" in + "ERROR") + echo "[$timestamp] ERROR: $message" >&2 + ;; + "WARN") + echo "[$timestamp] WARN: $message" >&2 + ;; + "INFO") + echo "[$timestamp] INFO: $message" + ;; + "DEBUG") + [[ "${DEBUG:-false}" == "true" ]] && echo "[$timestamp] DEBUG: $message" + ;; + *) + echo "[$timestamp] $message" + ;; + esac +} + +# Validation functions +function is_valid_email() { + local email="$1" + [[ "$email" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]] +} + +function is_number() { + [[ $1 =~ ^[0-9]+$ ]] +} diff --git a/test/solidlsp/bash/__init__.py b/test/solidlsp/bash/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py new file mode 100644 index 00000000..3fd1f637 --- /dev/null +++ b/test/solidlsp/bash/test_bash_basic.py @@ -0,0 +1,137 @@ +""" +Basic integration tests for the bash language server functionality. + +These tests validate the functionality of the language server APIs +like request_document_symbols using the bash test repository. +""" + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +@pytest.mark.bash +class TestBashLanguageServerBasics: + """Test basic functionality of the bash language server.""" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_language_server_initialization(self, language_server: SolidLanguageServer) -> None: + """Test that bash language server can be initialized successfully.""" + assert language_server is not None + assert language_server.language == Language.BASH + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_request_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols for bash files.""" + # Test getting symbols from main.sh + all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + + # Extract function symbols (LSP Symbol Kind 12) + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] + function_names = [symbol["name"] for symbol in function_symbols] + + # Should detect all 3 functions from main.sh + assert "greet_user" in function_names, "Should find greet_user function" + assert "process_items" in function_names, "Should find process_items function" + assert "main" in function_names, "Should find main function" + assert len(function_symbols) >= 3, f"Should find at least 3 functions, found {len(function_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols with body extraction.""" + # Test with include_body=True + all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=True) + + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] + + # Find greet_user function and check it has body + greet_user_symbol = next((sym for sym in function_symbols if sym["name"] == "greet_user"), None) + assert greet_user_symbol is not None, "Should find greet_user function" + + if "body" in greet_user_symbol: + body = greet_user_symbol["body"] + assert "function greet_user()" in body, "Function body should contain function definition" + assert "case" in body.lower(), "Function body should contain case statement" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: + """Test function detection in utils.sh file.""" + # Test with utils.sh as well + utils_all_symbols, utils_root_symbols = language_server.request_document_symbols( + "utils.sh", include_body=False + ) + + utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] + utils_function_names = [symbol["name"] for symbol in utils_function_symbols] + + # Should detect functions from utils.sh + expected_utils_functions = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", + "contains_element", "log_message", "is_valid_email", "is_number"] + + for func_name in expected_utils_functions: + assert func_name in utils_function_names, f"Should find {func_name} function in utils.sh" + + assert len(utils_function_symbols) >= 8, f"Should find at least 8 functions in utils.sh, found {len(utils_function_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_regex_fallback_detection(self, language_server: SolidLanguageServer) -> None: + """Test the regex-based fallback function detection directly.""" + # Check if the method exists (it should if we have our enhanced bash server) + if not hasattr(language_server, '_detect_bash_functions'): + pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") + + # Test the _detect_bash_functions method directly + detected_functions = language_server._detect_bash_functions("main.sh", include_body=True) + + # Should detect functions even if LSP doesn't provide them + assert len(detected_functions) == 3, f"Should detect exactly 3 functions, found {len(detected_functions)}" + + function_names = [func["name"] for func in detected_functions] + assert "greet_user" in function_names, "Regex should detect greet_user function" + assert "process_items" in function_names, "Regex should detect process_items function" + assert "main" in function_names, "Regex should detect main function" + + # Check that symbols have proper structure + for func in detected_functions: + assert func["kind"] == 12, f"Function {func['name']} should have LSP Symbol Kind 12" + assert "location" in func, f"Function {func['name']} should have location info" + assert "range" in func, f"Function {func['name']} should have range info" + assert "selectionRange" in func, f"Function {func['name']} should have selectionRange info" + + # Check location structure + location = func["location"] + assert "uri" in location, f"Function {func['name']} location should have URI" + assert "range" in location, f"Function {func['name']} location should have range" + assert "relativePath" in location, f"Function {func['name']} location should have relative path" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: + """Test detection of different bash function syntax patterns.""" + # Check if the method exists (it should if we have our enhanced bash server) + if not hasattr(language_server, '_detect_bash_functions'): + pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") + + # Test main.sh (has 'function' keyword functions and regular function) + main_functions = language_server._detect_bash_functions("main.sh", include_body=False) + + # Test utils.sh (all use 'function' keyword) + utils_functions = language_server._detect_bash_functions("utils.sh", include_body=False) + + # Verify we detect both syntax patterns + main_function_names = [func["name"] for func in main_functions] + utils_function_names = [func["name"] for func in utils_functions] + + # main() uses regular syntax: main() { + assert "main" in main_function_names, "Should detect regular function syntax" + + # Functions with 'function' keyword: function name() { + assert "greet_user" in main_function_names, "Should detect function keyword syntax" + assert "to_uppercase" in utils_function_names, "Should detect function keyword syntax in utils" + + # Verify all expected utils functions are detected + expected_utils = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", + "contains_element", "log_message", "is_valid_email", "is_number"] + + for expected_func in expected_utils: + assert expected_func in utils_function_names, f"Should detect {expected_func} function" \ No newline at end of file From f2d81812c787fab44ac4b8c0f8539401efc839cb Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 6 Aug 2025 20:58:14 +0200 Subject: [PATCH 046/220] Apply formatter --- .../language_servers/bash_language_server.py | 110 +++++++++--------- test/conftest.py | 3 - test/solidlsp/bash/test_bash_basic.py | 82 +++++++------ test/solidlsp/ruby/test_ruby_basic.py | 12 +- 4 files changed, 107 insertions(+), 100 deletions(-) diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py index 04ffe355..2b8d8549 100644 --- a/src/solidlsp/language_servers/bash_language_server.py +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -9,8 +9,7 @@ import pathlib import shutil import threading -from overrides import override - +from solidlsp import ls_types from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig @@ -20,8 +19,6 @@ from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings -from solidlsp import ls_types - class BashLanguageServer(SolidLanguageServer): """ Provides Bash specific instantiation of the LanguageServer class using bash-language-server. @@ -47,8 +44,6 @@ class BashLanguageServer(SolidLanguageServer): self.server_ready = threading.Event() self.initialize_searcher_command_available = threading.Event() - - @classmethod def _setup_runtime_dependencies( cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings @@ -180,7 +175,7 @@ class BashLanguageServer(SolidLanguageServer): # Enhanced capability checks for bash-language-server 5.6.0 assert init_response["capabilities"]["textDocumentSync"] in [1, 2] # Full or Incremental assert "completionProvider" in init_response["capabilities"] - + # Verify document symbol support is available if "documentSymbolProvider" in init_response["capabilities"]: self.logger.log("Bash server supports document symbols", logging.INFO) @@ -204,139 +199,138 @@ class BashLanguageServer(SolidLanguageServer): ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: """ Enhanced document symbol request with hybrid LSP + regex-based function detection for bash files. - + This method combines both LSP-based detection and regex-based detection to provide comprehensive function discovery. This dual approach is necessary because: - + 1. bash-language-server (v5.6.0) has inconsistent function detection capabilities 2. Some bash function syntaxes are not reliably detected by the LSP server 3. Files may contain mixed function notation styles within the same file 4. Different formatting or indentation can affect LSP detection - + The hybrid approach ensures maximum compatibility and comprehensive function discovery for reliable symbolic editing operations in Serena. """ # First get symbols from the standard LSP approach lsp_all_symbols, lsp_root_symbols = super().request_document_symbols(relative_file_path, include_body) - + # Always run regex-based function detection for comprehensive coverage # This addresses bash-language-server limitations and ensures we catch all function patterns self.logger.log(f"Running hybrid function detection (LSP + regex) for {relative_file_path}", logging.DEBUG) - + regex_detected_functions = self._detect_bash_functions(relative_file_path, include_body) - + # Merge LSP and regex results, avoiding duplicates merged_all_symbols, merged_root_symbols = self._merge_function_detections( lsp_all_symbols, lsp_root_symbols, regex_detected_functions ) - + # Log detection results for debugging lsp_functions = [s for s in lsp_all_symbols if s.get("kind") == 12] total_functions = [s for s in merged_all_symbols if s.get("kind") == 12] - + self.logger.log( f"Function detection for {relative_file_path}: LSP={len(lsp_functions)}, " - f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", - logging.INFO + f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", + logging.INFO, ) - + return merged_all_symbols, merged_root_symbols def _merge_function_detections( - self, - lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], + self, + lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], lsp_root_symbols: list[ls_types.UnifiedSymbolInformation], - regex_detected_functions: list[ls_types.UnifiedSymbolInformation] + regex_detected_functions: list[ls_types.UnifiedSymbolInformation], ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: """ Merge LSP-detected symbols with regex-detected functions, avoiding duplicates. - + This method: 1. Keeps all non-function symbols from LSP detection 2. Keeps all LSP-detected functions (they have more accurate positioning) 3. Adds regex-detected functions that weren't found by LSP 4. Uses function names to detect duplicates - + Args: lsp_all_symbols: All symbols detected by LSP lsp_root_symbols: Root-level symbols detected by LSP regex_detected_functions: Functions detected by regex (all are root-level) - + Returns: Tuple of (merged_all_symbols, merged_root_symbols) + """ # Extract function names that LSP already detected lsp_function_names = set() for symbol in lsp_all_symbols: if symbol.get("kind") == 12: # LSP Symbol Kind 12 = Function lsp_function_names.add(symbol["name"]) - + # Start with all LSP symbols (both functions and non-functions) merged_all_symbols = lsp_all_symbols.copy() merged_root_symbols = lsp_root_symbols.copy() - + # Add regex-detected functions that weren't found by LSP added_functions = 0 for regex_function in regex_detected_functions: function_name = regex_function["name"] - + # Only add if this function wasn't detected by LSP if function_name not in lsp_function_names: merged_all_symbols.append(regex_function) merged_root_symbols.append(regex_function) # All regex functions are root-level added_functions += 1 self.logger.log(f"Added regex-detected function '{function_name}' not found by LSP", logging.DEBUG) - + if added_functions > 0: self.logger.log(f"Merged {added_functions} additional functions from regex detection", logging.INFO) - + return merged_all_symbols, merged_root_symbols - + def _detect_bash_functions(self, relative_file_path: str, include_body: bool = False) -> list[ls_types.UnifiedSymbolInformation]: """ Regex-based detection of bash functions as fallback when LSP doesn't provide them. """ import re + from solidlsp import ls_types - + try: # Read the file content directly from filesystem abs_path = os.path.join(self.repository_root_path, relative_file_path) - with open(abs_path, 'r', encoding='utf-8') as f: + with open(abs_path, encoding="utf-8") as f: file_content = f.read() lines = file_content.split("\n") - + detected_functions = [] - + # Regex patterns for bash function definitions # Pattern 1: function name() { ... } # Pattern 2: name() { ... } function_patterns = [ - re.compile(r'^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{'), - re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{') + re.compile(r"^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{"), + re.compile(r"^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{"), ] - + for line_num, line in enumerate(lines): for pattern in function_patterns: match = pattern.match(line) if match: func_name = match.group(1) - + # Find the end of the function by matching braces start_line = line_num end_line = self._find_function_end(lines, start_line) - + # Create location information location = ls_types.Location( uri=pathlib.Path(abs_path).as_uri(), - range={ - "start": {"line": start_line, "character": match.start(1)}, - "end": {"line": end_line, "character": 0} - }, + range={"start": {"line": start_line, "character": match.start(1)}, "end": {"line": end_line, "character": 0}}, absolutePath=abs_path, - relativePath=relative_file_path + relativePath=relative_file_path, ) - + # Create the function symbol func_symbol = ls_types.UnifiedSymbolInformation( name=func_name, @@ -345,44 +339,44 @@ class BashLanguageServer(SolidLanguageServer): range=location["range"], selectionRange={ "start": {"line": start_line, "character": match.start(1)}, - "end": {"line": start_line, "character": match.end(1)} + "end": {"line": start_line, "character": match.end(1)}, }, children=[], - parent=None + parent=None, ) - + # Add function body if requested if include_body: - func_symbol["body"] = "\n".join(lines[start_line:end_line + 1]) - + func_symbol["body"] = "\n".join(lines[start_line : end_line + 1]) + detected_functions.append(func_symbol) break - + return detected_functions - + except Exception as e: self.logger.log(f"Error in regex-based function detection for {relative_file_path}: {e}", logging.WARNING) return [] - + def _find_function_end(self, lines: list[str], start_line: int) -> int: """ Find the end line of a bash function by matching opening and closing braces. """ brace_count = 0 in_function = False - + for i in range(start_line, len(lines)): line = lines[i] - + # Count braces, handling basic cases for char in line: - if char == '{': + if char == "{": brace_count += 1 in_function = True - elif char == '}' and in_function: + elif char == "}" and in_function: brace_count -= 1 if brace_count == 0: return i - + # Fallback: if we can't match braces properly, assume single line or small function return min(start_line + 10, len(lines) - 1) diff --git a/test/conftest.py b/test/conftest.py index da91ebdd..849af543 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -144,6 +144,3 @@ def project(request: LanguageParamRequest): language = request.param yield create_default_project(language) - - - diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py index 3fd1f637..8db362e9 100644 --- a/test/solidlsp/bash/test_bash_basic.py +++ b/test/solidlsp/bash/test_bash_basic.py @@ -26,14 +26,14 @@ class TestBashLanguageServerBasics: """Test request_document_symbols for bash files.""" # Test getting symbols from main.sh all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=False) - + # Extract function symbols (LSP Symbol Kind 12) function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] function_names = [symbol["name"] for symbol in function_symbols] - + # Should detect all 3 functions from main.sh assert "greet_user" in function_names, "Should find greet_user function" - assert "process_items" in function_names, "Should find process_items function" + assert "process_items" in function_names, "Should find process_items function" assert "main" in function_names, "Should find main function" assert len(function_symbols) >= 3, f"Should find at least 3 functions, found {len(function_symbols)}" @@ -42,13 +42,13 @@ class TestBashLanguageServerBasics: """Test request_document_symbols with body extraction.""" # Test with include_body=True all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=True) - + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] - + # Find greet_user function and check it has body greet_user_symbol = next((sym for sym in function_symbols if sym["name"] == "greet_user"), None) assert greet_user_symbol is not None, "Should find greet_user function" - + if "body" in greet_user_symbol: body = greet_user_symbol["body"] assert "function greet_user()" in body, "Function body should contain function definition" @@ -58,47 +58,53 @@ class TestBashLanguageServerBasics: def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: """Test function detection in utils.sh file.""" # Test with utils.sh as well - utils_all_symbols, utils_root_symbols = language_server.request_document_symbols( - "utils.sh", include_body=False - ) - + utils_all_symbols, utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [symbol["name"] for symbol in utils_function_symbols] - + # Should detect functions from utils.sh - expected_utils_functions = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", - "contains_element", "log_message", "is_valid_email", "is_number"] - + expected_utils_functions = [ + "to_uppercase", + "to_lowercase", + "trim_whitespace", + "backup_file", + "contains_element", + "log_message", + "is_valid_email", + "is_number", + ] + for func_name in expected_utils_functions: assert func_name in utils_function_names, f"Should find {func_name} function in utils.sh" - + assert len(utils_function_symbols) >= 8, f"Should find at least 8 functions in utils.sh, found {len(utils_function_symbols)}" - @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) def test_bash_regex_fallback_detection(self, language_server: SolidLanguageServer) -> None: """Test the regex-based fallback function detection directly.""" # Check if the method exists (it should if we have our enhanced bash server) - if not hasattr(language_server, '_detect_bash_functions'): + if not hasattr(language_server, "_detect_bash_functions"): pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") - - # Test the _detect_bash_functions method directly + + # Test the _detect_bash_functions method directly detected_functions = language_server._detect_bash_functions("main.sh", include_body=True) - + # Should detect functions even if LSP doesn't provide them assert len(detected_functions) == 3, f"Should detect exactly 3 functions, found {len(detected_functions)}" - + function_names = [func["name"] for func in detected_functions] assert "greet_user" in function_names, "Regex should detect greet_user function" assert "process_items" in function_names, "Regex should detect process_items function" assert "main" in function_names, "Regex should detect main function" - + # Check that symbols have proper structure for func in detected_functions: assert func["kind"] == 12, f"Function {func['name']} should have LSP Symbol Kind 12" assert "location" in func, f"Function {func['name']} should have location info" assert "range" in func, f"Function {func['name']} should have range info" assert "selectionRange" in func, f"Function {func['name']} should have selectionRange info" - + # Check location structure location = func["location"] assert "uri" in location, f"Function {func['name']} location should have URI" @@ -109,29 +115,37 @@ class TestBashLanguageServerBasics: def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: """Test detection of different bash function syntax patterns.""" # Check if the method exists (it should if we have our enhanced bash server) - if not hasattr(language_server, '_detect_bash_functions'): + if not hasattr(language_server, "_detect_bash_functions"): pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") - + # Test main.sh (has 'function' keyword functions and regular function) main_functions = language_server._detect_bash_functions("main.sh", include_body=False) - + # Test utils.sh (all use 'function' keyword) utils_functions = language_server._detect_bash_functions("utils.sh", include_body=False) - + # Verify we detect both syntax patterns main_function_names = [func["name"] for func in main_functions] utils_function_names = [func["name"] for func in utils_functions] - + # main() uses regular syntax: main() { assert "main" in main_function_names, "Should detect regular function syntax" - + # Functions with 'function' keyword: function name() { assert "greet_user" in main_function_names, "Should detect function keyword syntax" assert "to_uppercase" in utils_function_names, "Should detect function keyword syntax in utils" - + # Verify all expected utils functions are detected - expected_utils = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", - "contains_element", "log_message", "is_valid_email", "is_number"] - + expected_utils = [ + "to_uppercase", + "to_lowercase", + "trim_whitespace", + "backup_file", + "contains_element", + "log_message", + "is_valid_email", + "is_number", + ] + for expected_func in expected_utils: - assert expected_func in utils_function_names, f"Should detect {expected_func} function" \ No newline at end of file + assert expected_func in utils_function_names, f"Should detect {expected_func} function" diff --git a/test/solidlsp/ruby/test_ruby_basic.py b/test/solidlsp/ruby/test_ruby_basic.py index 435c0538..2edf95f2 100644 --- a/test/solidlsp/ruby/test_ruby_basic.py +++ b/test/solidlsp/ruby/test_ruby_basic.py @@ -27,14 +27,16 @@ class TestRubyLanguageServer: helper_symbol = sym break print(helper_symbol) - assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" + assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) - @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) - def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: + @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) + def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: # Test finding Calculator.add method definition from line 17: Calculator.new.add(demo.value, 10) - definition_location_list = language_server.request_definition(str(repo_path / "main.rb"), 16,17) # add method at line 17 (0-indexed 16), position 17 - + definition_location_list = language_server.request_definition( + str(repo_path / "main.rb"), 16, 17 + ) # add method at line 17 (0-indexed 16), position 17 + assert len(definition_location_list) == 1 definition_location = definition_location_list[0] print(f"Found definition: {definition_location}") From 2b4d68bea940f9d58c7f938ed7248e4aa921ca22 Mon Sep 17 00:00:00 2001 From: Akira TSURUDA Date: Thu, 7 Aug 2025 07:05:35 +0900 Subject: [PATCH 047/220] Add Bundler support and timeout configuration for Ruby projects (#419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Bundler support for Ruby projects This change enhances the Solargraph language server implementation to properly support Ruby projects using Bundler with vendor/bundle configurations. Key improvements: - Detects Bundler projects by checking for Gemfile existence - Supports both 'bin/bundle' and 'bundle' commands - Checks Gemfile.lock for solargraph availability - Uses 'bundle exec solargraph' when appropriate - Provides clear error messages for missing solargraph in Bundler projects - Maintains backward compatibility with non-Bundler projects Fixes permission issues encountered with rbenv/system-managed Ruby installations when trying to install gems globally. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude * Add timeout configuration for Solargraph language server This change adds proper timeout handling to the Solargraph Ruby language server to address initialization timeout issues, especially in Bundler environments. Key improvements: - Set 120-second request timeout for LSP requests and initialization - Add 60-second server readiness timeout with graceful fallback - Enhanced logging for timeout scenarios - Better handling of slow initialization in large projects or Bundler environments This resolves connection timeout issues where Solargraph takes longer than the default 30-second MCP timeout to fully initialize and become ready. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --------- Co-authored-by: Claude --- src/solidlsp/language_servers/solargraph.py | 108 ++++++++++++++++---- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index ca148f14..b75b19e9 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -48,6 +48,9 @@ class Solargraph(SolidLanguageServer): self.initialize_searcher_command_available = threading.Event() self.resolve_main_method_available = threading.Event() + # Set timeout for Solargraph requests - Bundler environments may need more time + self.set_request_timeout(120.0) # 120 seconds for initialization and requests + @override def is_ignored_dirname(self, dirname: str) -> bool: return super().is_ignored_dirname(dirname) or dirname in ["vendor"] @@ -67,35 +70,84 @@ class Solargraph(SolidLanguageServer): except FileNotFoundError as e: raise RuntimeError("Ruby is not installed. Please install Ruby before continuing.") from e - # Check if solargraph is installed + # Check for Bundler project (Gemfile exists) + gemfile_path = os.path.join(repository_root_path, "Gemfile") + gemfile_lock_path = os.path.join(repository_root_path, "Gemfile.lock") + is_bundler_project = os.path.exists(gemfile_path) + + if is_bundler_project: + logger.log("Detected Bundler project (Gemfile found)", logging.INFO) + + # Check if bundle command is available + bundle_path = shutil.which("bundle") + if not bundle_path: + # Try common bundle executables + for bundle_cmd in ["bin/bundle", "bundle"]: + bundle_full_path = ( + os.path.join(repository_root_path, bundle_cmd) if bundle_cmd.startswith("bin/") else shutil.which(bundle_cmd) + ) + if bundle_full_path and os.path.exists(bundle_full_path): + bundle_path = bundle_full_path if bundle_cmd.startswith("bin/") else bundle_cmd + break + + if not bundle_path: + raise RuntimeError("Bundler project detected but 'bundle' command not found. Please install Bundler.") + + # Check if solargraph is in Gemfile.lock + solargraph_in_bundle = False + if os.path.exists(gemfile_lock_path): + try: + with open(gemfile_lock_path) as f: + content = f.read() + solargraph_in_bundle = "solargraph" in content.lower() + except Exception as e: + logger.log(f"Warning: Could not read Gemfile.lock: {e}", logging.WARNING) + + if solargraph_in_bundle: + logger.log("Found solargraph in Gemfile.lock", logging.INFO) + return f"{bundle_path} exec solargraph" + else: + logger.log( + "solargraph not found in Gemfile.lock. Please add 'gem \"solargraph\"' to your Gemfile and run 'bundle install'", + logging.WARNING, + ) + # Fall through to global installation check + + # Check if solargraph is installed globally # First, try to find solargraph in PATH (includes asdf shims) solargraph_path = shutil.which("solargraph") if solargraph_path: logger.log(f"Found solargraph at: {solargraph_path}", logging.INFO) return solargraph_path - # Fallback to gem exec - runtime_dependencies = [ - { - "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", - "installCommand": "gem install solargraph -v 0.51.1", - "binaryName": "solargraph", - "archiveType": "gem", - } - ] + # Fallback to gem exec (for non-Bundler projects or when global solargraph not found) + if not is_bundler_project: + runtime_dependencies = [ + { + "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", + "installCommand": "gem install solargraph -v 0.51.1", + "binaryName": "solargraph", + "archiveType": "gem", + } + ] - dependency = runtime_dependencies[0] - try: - result = subprocess.run( - ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path + dependency = runtime_dependencies[0] + try: + result = subprocess.run( + ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path + ) + if result.stdout.strip() == "false": + logger.log("Installing Solargraph...", logging.INFO) + subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) + + return "gem exec solargraph" + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e + else: + raise RuntimeError( + "This appears to be a Bundler project, but solargraph is not available. " + "Please add 'gem \"solargraph\"' to your Gemfile and run 'bundle install'." ) - if result.stdout.strip() == "false": - logger.log("Installing Solargraph...", logging.INFO) - subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) - - return "gem exec solargraph" - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: @@ -176,4 +228,16 @@ class Solargraph(SolidLanguageServer): self.completions_available.set() self.server_ready.set() - self.server_ready.wait() + + # Wait for server to be ready with timeout, especially important for Bundler environments + server_ready_timeout = 60.0 + self.logger.log(f"Waiting up to {server_ready_timeout} seconds for Solargraph to become ready...", logging.INFO) + + if self.server_ready.wait(timeout=server_ready_timeout): + self.logger.log("Solargraph is ready and available for requests", logging.INFO) + else: + self.logger.log( + f"Timeout waiting for Solargraph to become ready within {server_ready_timeout} seconds, proceeding anyway. " + "This may indicate slow initialization in Bundler environment or large project indexing.", + logging.WARNING, + ) From b09cbc089da79731f17a99d7c800ae0b7a783a4e Mon Sep 17 00:00:00 2001 From: johngambleubind <127568872+johngambleubind@users.noreply.github.com> Date: Thu, 7 Aug 2025 18:52:01 +1000 Subject: [PATCH 048/220] Fix #384: Use safe zip extraction for C# dependencies (#418) --- .../csharp_language_server.py | 10 +- src/solidlsp/util/zip.py | 127 ++++++++++++++++++ test/solidlsp/util/test_zip.py | 103 ++++++++++++++ 3 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 src/solidlsp/util/zip.py create mode 100644 test/solidlsp/util/test_zip.py diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index f01d5b04..b8f0cc38 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -25,6 +25,7 @@ from solidlsp.ls_utils import PathUtils from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings +from solidlsp.util.zip import SafeZipExtractor from .common import RuntimeDependency @@ -386,8 +387,13 @@ class CSharpLanguageServer(SolidLanguageServer): package_extract_dir = temp_dir / f"{package_name}.{package_version}" package_extract_dir.mkdir(exist_ok=True) - with zipfile.ZipFile(nupkg_file, "r") as zip_ref: - zip_ref.extractall(package_extract_dir) + # Use SafeZipExtractor to handle long paths and skip errors + extractor = SafeZipExtractor( + archive_path=nupkg_file, + extract_dir=package_extract_dir, + verbose=False + ) + extractor.extract_all() # Clean up the nupkg file nupkg_file.unlink() diff --git a/src/solidlsp/util/zip.py b/src/solidlsp/util/zip.py new file mode 100644 index 00000000..ae0c3183 --- /dev/null +++ b/src/solidlsp/util/zip.py @@ -0,0 +1,127 @@ +import logging +import os +import sys +import zipfile +from pathlib import Path +from typing import Optional, List +import fnmatch + +log = logging.getLogger(__name__) + +class SafeZipExtractor: + """ + A utility class for extracting ZIP archives safely. + + Features: + - Handles long file paths on Windows + - Skips files that fail to extract, continuing with the rest + - Creates necessary directories automatically + - Optional include/exclude pattern filters + """ + + def __init__( + self, + archive_path: Path, + extract_dir: Path, + verbose: bool = True, + include_patterns: Optional[List[str]] = None, + exclude_patterns: Optional[List[str]] = None + ) -> None: + """ + Initialize the SafeZipExtractor. + + :param archive_path: Path to the ZIP archive file + :param extract_dir: Directory where files will be extracted + :param verbose: Whether to log status messages + :param include_patterns: List of glob patterns for files to extract (None = all files) + :param exclude_patterns: List of glob patterns for files to skip + """ + self.archive_path = Path(archive_path) + self.extract_dir = Path(extract_dir) + self.verbose = verbose + self.include_patterns = include_patterns or [] + self.exclude_patterns = exclude_patterns or [] + + def extract_all(self) -> None: + """ + Extract all files from the archive, skipping any that fail. + """ + if not self.archive_path.exists(): + raise FileNotFoundError(f"Archive not found: {self.archive_path}") + + if self.verbose: + log.info(f"Extracting from: {self.archive_path} to {self.extract_dir}") + + with zipfile.ZipFile(self.archive_path, "r") as zip_ref: + for member in zip_ref.infolist(): + if self._should_extract(member.filename): + self._extract_member(zip_ref, member) + elif self.verbose: + log.info(f"Skipped: {member.filename}") + + def _should_extract(self, filename: str) -> bool: + """ + Determine whether a file should be extracted based on include/exclude patterns. + + :param filename: The file name from the archive + :return: True if the file should be extracted + """ + # If include_patterns is set, only extract if it matches at least one pattern + if self.include_patterns: + if not any(fnmatch.fnmatch(filename, pattern) for pattern in self.include_patterns): + return False + + # If exclude_patterns is set, skip if it matches any pattern + if self.exclude_patterns: + if any(fnmatch.fnmatch(filename, pattern) for pattern in self.exclude_patterns): + return False + + return True + + def _extract_member(self, zip_ref: zipfile.ZipFile, member: zipfile.ZipInfo) -> None: + """ + Extract a single member from the archive with error handling. + + :param zip_ref: Open ZipFile object + :param member: ZipInfo object representing the file + """ + try: + target_path = self.extract_dir / member.filename + + # Ensure directory structure exists + target_path.parent.mkdir(parents=True, exist_ok=True) + + # Handle long paths on Windows + final_path = self._normalize_path(target_path) + + # Extract file + with zip_ref.open(member) as source, open(final_path, "wb") as target: + target.write(source.read()) + + if self.verbose: + log.info(f"Extracted: {member.filename}") + + except Exception as e: + log.error(f"Failed to extract {member.filename}: {e}") + + @staticmethod + def _normalize_path(path: Path) -> Path: + """ + Adjust path to handle long paths on Windows. + + :param path: Original path + :return: Normalized path + """ + if sys.platform.startswith("win"): + return Path(r"\\?\{}".format(os.path.abspath(path))) + return path + + +# Example usage: +# extractor = SafeZipExtractor( +# archive_path=Path("file.nupkg"), +# extract_dir=Path("extract_dir"), +# include_patterns=["*.dll", "*.xml"], +# exclude_patterns=["*.pdb"] +# ) +# extractor.extract_all() diff --git a/test/solidlsp/util/test_zip.py b/test/solidlsp/util/test_zip.py new file mode 100644 index 00000000..4498781a --- /dev/null +++ b/test/solidlsp/util/test_zip.py @@ -0,0 +1,103 @@ +import io +import os +import sys +import zipfile +from pathlib import Path +import pytest + +from solidlsp.util.zip import SafeZipExtractor + + +@pytest.fixture +def temp_zip_file(tmp_path: Path) -> Path: + """Create a temporary ZIP file for testing.""" + zip_path = tmp_path / "test.zip" + with zipfile.ZipFile(zip_path, "w") as zipf: + zipf.writestr("file1.txt", "Hello World 1") + zipf.writestr("file2.txt", "Hello World 2") + zipf.writestr("folder/file3.txt", "Hello World 3") + return zip_path + + +def test_extract_all_success(temp_zip_file: Path, tmp_path: Path) -> None: + """All files should extract without error.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + extractor.extract_all() + + assert (dest_dir / "file1.txt").read_text() == "Hello World 1" + assert (dest_dir / "file2.txt").read_text() == "Hello World 2" + assert (dest_dir / "folder" / "file3.txt").read_text() == "Hello World 3" + + +def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Only files matching include_patterns should be extracted.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"] + ) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Files matching exclude_patterns should be skipped.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"] + ) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert not (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_include_and_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Exclude should override include if both match.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, + dest_dir, + verbose=False, + include_patterns=["*.txt"], + exclude_patterns=["file1.txt"], + ) + extractor.extract_all() + + assert not (dest_dir / "file1.txt").exists() + assert (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None: + """Should skip a file that raises an error and continue extracting others.""" + dest_dir = tmp_path / "extracted" + + original_open = zipfile.ZipFile.open + + def failing_open(self, member, *args, **kwargs): + if member.filename == "file2.txt": + raise IOError("Simulated failure") + return original_open(self, member, *args, **kwargs) + + # Patch the method on the class, not on an instance + monkeypatch.setattr(zipfile.ZipFile, "open", failing_open) + + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert not (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + +@pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows-only test") +def test_long_path_normalization(temp_zip_file: Path, tmp_path: Path) -> None: + """Ensure _normalize_path adds \\?\ prefix on Windows.""" + dest_dir = tmp_path / ("a" * 250) # Simulate long path + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + norm_path = extractor._normalize_path(dest_dir / "file.txt") + assert str(norm_path).startswith("\\\\?\\") From eaf863df6d60cbcad680403a51d1230e0386f825 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 6 Aug 2025 20:58:54 +0200 Subject: [PATCH 049/220] Apply platform-specific default kwargs to subprocess calls For now, we only set creationflags=CREATE_NO_WINDOW on Windows --- src/serena/cli.py | 10 ++++++---- src/serena/util/shell.py | 13 ++++--------- src/solidlsp/language_servers/common.py | 7 +++---- src/solidlsp/ls_handler.py | 5 ++++- src/solidlsp/util/subprocess_util.py | 13 +++++++++++++ 5 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 src/solidlsp/util/subprocess_util.py diff --git a/src/serena/cli.py b/src/serena/cli.py index 457b3acb..9927390c 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -31,6 +31,7 @@ from serena.project import Project from serena.tools import ToolRegistry from serena.util.logging import MemoryLogHandler from solidlsp.ls_config import Language +from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) @@ -40,18 +41,19 @@ log = logging.getLogger(__name__) def _open_in_editor(path: str) -> None: """Open the given file in the system's default editor or viewer.""" editor = os.environ.get("EDITOR") + run_kwargs = subprocess_kwargs() try: if editor: - subprocess.run([editor, path], check=False) + subprocess.run([editor, path], check=False, **run_kwargs) elif sys.platform.startswith("win"): try: os.startfile(path) except OSError: - subprocess.run(["notepad.exe", path], check=False) + subprocess.run(["notepad.exe", path], check=False, **run_kwargs) elif sys.platform == "darwin": - subprocess.run(["open", path], check=False) + subprocess.run(["open", path], check=False, **run_kwargs) else: - subprocess.run(["xdg-open", path], check=False) + subprocess.run(["xdg-open", path], check=False, **run_kwargs) except Exception as e: print(f"Failed to open {path}: {e}") diff --git a/src/serena/util/shell.py b/src/serena/util/shell.py index c899b212..59bc006f 100644 --- a/src/serena/util/shell.py +++ b/src/serena/util/shell.py @@ -4,6 +4,8 @@ import subprocess from pydantic import BaseModel +from solidlsp.util.subprocess_util import subprocess_kwargs + class ShellCommandResult(BaseModel): stdout: str @@ -36,6 +38,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr: encoding="utf-8", errors="replace", cwd=cwd, + **subprocess_kwargs(), ) stdout, stderr = process.communicate() @@ -43,15 +46,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr: def subprocess_check_output(args: list[str], encoding: str = "utf-8", strip: bool = True, timeout: float | None = None) -> str: - kwargs = { - "stdin": subprocess.DEVNULL, - "stderr": subprocess.PIPE, - "timeout": timeout, - "env": os.environ.copy(), - } - if platform.system() == "Windows": - kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW # type: ignore - output = subprocess.check_output(args, **kwargs).decode(encoding) # type: ignore + output = subprocess.check_output(args, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout, env=os.environ.copy(), **subprocess_kwargs()).decode(encoding) # type: ignore if strip: output = output.strip() return output diff --git a/src/solidlsp/language_servers/common.py b/src/solidlsp/language_servers/common.py index 1587d0a9..4a1fa865 100644 --- a/src/solidlsp/language_servers/common.py +++ b/src/solidlsp/language_servers/common.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from solidlsp.ls_logger import LanguageServerLogger from solidlsp.ls_utils import FileUtils, PlatformUtils +from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) @@ -73,10 +74,8 @@ class RuntimeDependencyCollection: @staticmethod def _run_command(command: str | list[str], cwd: str) -> None: - kwargs = {} - if PlatformUtils.get_platform_id().is_windows(): - kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW # type: ignore - else: + kwargs = subprocess_kwargs() + if not PlatformUtils.get_platform_id().is_windows(): import pwd kwargs["user"] = pwd.getpwuid(os.getuid()).pw_name diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 6a994ed3..f81dde53 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -32,6 +32,7 @@ from solidlsp.lsp_protocol_handler.server import ( make_request, make_response, ) +from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) @@ -181,6 +182,8 @@ class SolidLanguageServerHandler: # on Linux/macOS cmd = " ".join(cmd) log.info("Starting language server process via command: %s", self.process_launch_info.cmd) + kwargs = subprocess_kwargs() + kwargs["start_new_session"] = self.start_independent_lsp_process self.process = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -188,8 +191,8 @@ class SolidLanguageServerHandler: stderr=subprocess.PIPE, env=child_proc_env, cwd=self.process_launch_info.cwd, - start_new_session=self.start_independent_lsp_process, shell=True, + **kwargs, ) # Check if process terminated immediately diff --git a/src/solidlsp/util/subprocess_util.py b/src/solidlsp/util/subprocess_util.py new file mode 100644 index 00000000..ac6a4e2b --- /dev/null +++ b/src/solidlsp/util/subprocess_util.py @@ -0,0 +1,13 @@ +import platform +import subprocess + + +def subprocess_kwargs(): + """ + Returns a dictionary of keyword arguments for subprocess calls, adding platform-specific + flags that we want to use consistently. + """ + kwargs = {} + if platform.system() == "Windows": + kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW # type: ignore + return kwargs From e5641de0ff09c2b7ab7aac9514040550374b0bf0 Mon Sep 17 00:00:00 2001 From: "jetbrains-junie[bot]" <201638009+jetbrains-junie[bot]@users.noreply.github.com> Date: Thu, 7 Aug 2025 12:48:04 +0200 Subject: [PATCH 050/220] feat(junie): added .junie workflow (#430) Co-authored-by: jetbrains-junie[bot] <201638009+jetbrains-junie[bot]@users.noreply.github.com> --- .github/workflows/junie.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/junie.yml diff --git a/.github/workflows/junie.yml b/.github/workflows/junie.yml new file mode 100644 index 00000000..9860e79e --- /dev/null +++ b/.github/workflows/junie.yml @@ -0,0 +1,21 @@ +name: Junie +run-name: Junie run ${{ inputs.run_id }} + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + run_id: + description: "id of workflow process" + required: true + workflow_params: + description: "stringified params" + required: true + +jobs: + call-workflow-passing-data: + uses: jetbrains-junie/junie-workflows/.github/workflows/ej-issue.yml@main + with: + workflow_params: ${{ inputs.workflow_params }} From 0f4942900284a9560e1354af6f2c4a53f2442356 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 7 Aug 2025 12:55:11 +0200 Subject: [PATCH 051/220] Formatting --- .../language_servers/csharp_language_server.py | 6 +----- src/solidlsp/util/zip.py | 11 ++++++----- test/solidlsp/util/test_zip.py | 16 ++++++---------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index b8f0cc38..da65cb83 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -388,11 +388,7 @@ class CSharpLanguageServer(SolidLanguageServer): package_extract_dir.mkdir(exist_ok=True) # Use SafeZipExtractor to handle long paths and skip errors - extractor = SafeZipExtractor( - archive_path=nupkg_file, - extract_dir=package_extract_dir, - verbose=False - ) + extractor = SafeZipExtractor(archive_path=nupkg_file, extract_dir=package_extract_dir, verbose=False) extractor.extract_all() # Clean up the nupkg file diff --git a/src/solidlsp/util/zip.py b/src/solidlsp/util/zip.py index ae0c3183..d93d9421 100644 --- a/src/solidlsp/util/zip.py +++ b/src/solidlsp/util/zip.py @@ -1,13 +1,14 @@ +import fnmatch import logging import os import sys import zipfile from pathlib import Path -from typing import Optional, List -import fnmatch +from typing import Optional log = logging.getLogger(__name__) + class SafeZipExtractor: """ A utility class for extracting ZIP archives safely. @@ -24,8 +25,8 @@ class SafeZipExtractor: archive_path: Path, extract_dir: Path, verbose: bool = True, - include_patterns: Optional[List[str]] = None, - exclude_patterns: Optional[List[str]] = None + include_patterns: Optional[list[str]] = None, + exclude_patterns: Optional[list[str]] = None, ) -> None: """ Initialize the SafeZipExtractor. @@ -113,7 +114,7 @@ class SafeZipExtractor: :return: Normalized path """ if sys.platform.startswith("win"): - return Path(r"\\?\{}".format(os.path.abspath(path))) + return Path(rf"\\?\{os.path.abspath(path)}") return path diff --git a/test/solidlsp/util/test_zip.py b/test/solidlsp/util/test_zip.py index 4498781a..5fa545ba 100644 --- a/test/solidlsp/util/test_zip.py +++ b/test/solidlsp/util/test_zip.py @@ -1,8 +1,7 @@ -import io -import os import sys import zipfile from pathlib import Path + import pytest from solidlsp.util.zip import SafeZipExtractor @@ -33,9 +32,7 @@ def test_extract_all_success(temp_zip_file: Path, tmp_path: Path) -> None: def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: """Only files matching include_patterns should be extracted.""" dest_dir = tmp_path / "extracted" - extractor = SafeZipExtractor( - temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"] - ) + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"]) extractor.extract_all() assert (dest_dir / "file1.txt").exists() @@ -46,9 +43,7 @@ def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: def test_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: """Files matching exclude_patterns should be skipped.""" dest_dir = tmp_path / "extracted" - extractor = SafeZipExtractor( - temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"] - ) + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"]) extractor.extract_all() assert (dest_dir / "file1.txt").exists() @@ -81,7 +76,7 @@ def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None def failing_open(self, member, *args, **kwargs): if member.filename == "file2.txt": - raise IOError("Simulated failure") + raise OSError("Simulated failure") return original_open(self, member, *args, **kwargs) # Patch the method on the class, not on an instance @@ -94,9 +89,10 @@ def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None assert not (dest_dir / "file2.txt").exists() assert (dest_dir / "folder" / "file3.txt").exists() + @pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows-only test") def test_long_path_normalization(temp_zip_file: Path, tmp_path: Path) -> None: - """Ensure _normalize_path adds \\?\ prefix on Windows.""" + r"""Ensure _normalize_path adds \\?\\ prefix on Windows.""" dest_dir = tmp_path / ("a" * 250) # Simulate long path extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) norm_path = extractor._normalize_path(dest_dir / "file.txt") From c1d77012233d4f8b3e48f696c448008e9652b0e3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 7 Aug 2025 12:57:46 +0200 Subject: [PATCH 052/220] Add basic log level detection to stderr stream reader --- src/solidlsp/ls_handler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 6a994ed3..c1f50fa1 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -370,7 +370,13 @@ class SolidLanguageServerHandler: line = self.process.stderr.readline() if not line: continue - log.error(line.decode(ENCODING, errors="replace")) + line = line.decode(ENCODING, errors="replace") + line_lower = line.lower() + if "error" in line_lower or "exception" in line_lower or line.startswith("E["): + level = logging.ERROR + else: + level = logging.INFO + log.log(level, line) except Exception as e: log.error("Error while reading stderr from language server process: %s", e, exc_info=e) if not self._is_shutting_down: From 7912ba4d91319f99c6a7bb71f727777fd7f95afc Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 7 Aug 2025 13:31:34 +0200 Subject: [PATCH 053/220] Check formatting in CI --- .github/workflows/pytest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ddfe7a32..c5dd340f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -102,6 +102,9 @@ jobs: - name: Install dependencies shell: bash run: uv pip install -e ".[dev]" + - name: Check formatting + shell: bash + run: uv run poe lint - name: Test with pytest shell: bash run: uv run poe test From dc5be9fad40c8c2ccc186f6ca57995ebe4fe1ccd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 7 Aug 2025 13:41:45 +0200 Subject: [PATCH 054/220] Ignore ruff UP045 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8e4d563c..4363567b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -235,6 +235,7 @@ ignore = [ "SIM117", # forbids nested with statements "C400", # wants to unnecessarily force use of list comprehension "UP037", # can incorrectly (!) convert quoted type to unquoted type, causing an error + "UP045", # imposes T | None instead of Optional[T] ] unfixable = ["F841", "F601", "F602", "B018"] extend-fixable = ["F401", "B905", "W291"] From 68c5deeb1ad72d9a52d90a4457ccb01d8b6245fc Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 7 Aug 2025 17:37:27 +0200 Subject: [PATCH 055/220] Add missing closure-lsp dependency for osx-x64 --- src/solidlsp/language_servers/clojure_lsp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/solidlsp/language_servers/clojure_lsp.py b/src/solidlsp/language_servers/clojure_lsp.py index d81fa236..9e52f6d2 100644 --- a/src/solidlsp/language_servers/clojure_lsp.py +++ b/src/solidlsp/language_servers/clojure_lsp.py @@ -54,6 +54,13 @@ class ClojureLSP(SolidLanguageServer): archive_type="zip", binary_name="clojure-lsp", ), + RuntimeDependency( + id="clojure-lsp", + url=f"{clojure_lsp_releases}/clojure-lsp-native-macos-amd64.zip", + platform_id="osx-x64", + archive_type="zip", + binary_name="clojure-lsp", + ), RuntimeDependency( id="clojure-lsp", url=f"{clojure_lsp_releases}/clojure-lsp-native-linux-aarch64.zip", From 6d28c7d7223e0584a7b5a8ec00a36e5d9c56cc80 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 7 Aug 2025 17:34:51 +0200 Subject: [PATCH 056/220] Minor simplification of sourcekit_lsp.py, added sleeping at startup --- .../language_servers/sourcekit_lsp.py | 407 +++++------------- test/solidlsp/swift/test_swift_basic.py | 60 ++- 2 files changed, 125 insertions(+), 342 deletions(-) diff --git a/src/solidlsp/language_servers/sourcekit_lsp.py b/src/solidlsp/language_servers/sourcekit_lsp.py index dac6cfe4..f566b693 100644 --- a/src/solidlsp/language_servers/sourcekit_lsp.py +++ b/src/solidlsp/language_servers/sourcekit_lsp.py @@ -3,17 +3,16 @@ import os import pathlib import subprocess import threading +import time from overrides import override from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger -from solidlsp.lsp_protocol_handler import lsp_types from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings -from solidlsp.lsp_protocol_handler import lsp_types as LSPTypes class SourceKitLSP(SolidLanguageServer): @@ -31,53 +30,25 @@ class SourceKitLSP(SolidLanguageServer): return super().is_ignored_dirname(dirname) or dirname in [".build", ".swiftpm", "node_modules", "dist", "build"] @staticmethod - def _get_swift_version(): - """Get the installed Swift version or None if not found.""" - try: - result = subprocess.run(["swift", "--version"], capture_output=True, text=True, check=False) - if result.returncode == 0: - return result.stdout.strip() - except FileNotFoundError: - return None - return None - - @staticmethod - def _get_sourcekit_lsp_version(): - """Get the installed sourcekit-lsp version or None if not found.""" + def _get_sourcekit_lsp_version() -> str: + """Get the installed sourcekit-lsp version or raise error if sourcekit was not found.""" try: result = subprocess.run(["sourcekit-lsp", "-h"], capture_output=True, text=True, check=False) if result.returncode == 0: return result.stdout.strip() - except FileNotFoundError: - return None - return None - - @staticmethod - def _setup_runtime_dependency(): - """ - Check if required Swift runtime dependencies are available. - Raises RuntimeError with helpful message if dependencies are missing. - """ - swift_version = SourceKitLSP._get_swift_version() - if not swift_version: + else: + raise Exception(f"`sourcekit-lsp -h` resulted in: {result}") + except Exception as e: raise RuntimeError( - "Swift is not installed. Please install Swift from https://swift.org/download/ and make sure it is added to your PATH." - ) - - sourcekit_lsp_version = SourceKitLSP._get_sourcekit_lsp_version() - if not sourcekit_lsp_version: - raise RuntimeError( - "Found a Swift version but sourcekit-lsp is not installed.\n" - "Please install sourcekit-lsp as described in https://github.com/apple/sourcekit-lsp#installation\n\n" - "After installation, make sure it is added to your PATH." - ) - - return True + "Could not find sourcekit-lsp, please install it as described in https://github.com/apple/sourcekit-lsp#installation" + "And make sure it is available on your PATH." + ) from e def __init__( self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings ): - self._setup_runtime_dependency() + sourcekit_version = self._get_sourcekit_lsp_version() + logger.log(f"Starting sourcekit lsp with version: {sourcekit_version}", logging.INFO) super().__init__( config, @@ -96,40 +67,25 @@ class SourceKitLSP(SolidLanguageServer): Returns the initialize params for the Swift Language Server. """ root_uri = pathlib.Path(repository_absolute_path).as_uri() - + initialize_params = { "capabilities": { "general": { - "markdown": { - "parser": "marked", - "version": "1.1.0" - }, - "positionEncodings": [ - "utf-16" - ], - "regularExpressions": { - "engine": "ECMAScript", - "version": "ES2020" - }, + "markdown": {"parser": "marked", "version": "1.1.0"}, + "positionEncodings": ["utf-16"], + "regularExpressions": {"engine": "ECMAScript", "version": "ES2020"}, "staleRequestSupport": { "cancel": True, "retryOnContentModified": [ "textDocument/semanticTokens/full", "textDocument/semanticTokens/range", - "textDocument/semanticTokens/full/delta" - ] - } - }, - "notebookDocument": { - "synchronization": { - "dynamicRegistration": True, - "executionSummarySupport": True - } - }, - "textDocument": { - "callHierarchy": { - "dynamicRegistration": True + "textDocument/semanticTokens/full/delta", + ], }, + }, + "notebookDocument": {"synchronization": {"dynamicRegistration": True, "executionSummarySupport": True}}, + "textDocument": { + "callHierarchy": {"dynamicRegistration": True}, "codeAction": { "codeActionLiteralSupport": { "codeActionKind": { @@ -141,7 +97,7 @@ class SourceKitLSP(SolidLanguageServer): "refactor.inline", "refactor.rewrite", "source", - "source.organizeImports" + "source.organizeImports", ] } }, @@ -150,193 +106,85 @@ class SourceKitLSP(SolidLanguageServer): "dynamicRegistration": True, "honorsChangeAnnotations": True, "isPreferredSupport": True, - "resolveSupport": { - "properties": [ - "edit" - ] - } - }, - "codeLens": { - "dynamicRegistration": True - }, - "colorProvider": { - "dynamicRegistration": True + "resolveSupport": {"properties": ["edit"]}, }, + "codeLens": {"dynamicRegistration": True}, + "colorProvider": {"dynamicRegistration": True}, "completion": { "completionItem": { "commitCharactersSupport": True, "deprecatedSupport": True, - "documentationFormat": [ - "markdown", - "plaintext" - ], + "documentationFormat": ["markdown", "plaintext"], "insertReplaceSupport": True, - "insertTextModeSupport": { - "valueSet": [ - 1, - 2 - ] - }, + "insertTextModeSupport": {"valueSet": [1, 2]}, "labelDetailsSupport": True, "preselectSupport": True, - "resolveSupport": { - "properties": [ - "documentation", - "detail", - "additionalTextEdits" - ] - }, + "resolveSupport": {"properties": ["documentation", "detail", "additionalTextEdits"]}, "snippetSupport": True, - "tagSupport": { - "valueSet": [ - 1 - ] - } + "tagSupport": {"valueSet": [1]}, }, "completionItemKind": { - "valueSet": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 - ] - }, - "completionList": { - "itemDefaults": [ - "commitCharacters", - "editRange", - "insertTextFormat", - "insertTextMode", - "data" - ] + "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25] }, + "completionList": {"itemDefaults": ["commitCharacters", "editRange", "insertTextFormat", "insertTextMode", "data"]}, "contextSupport": True, "dynamicRegistration": True, - "insertTextMode": 2 - }, - "declaration": { - "dynamicRegistration": True, - "linkSupport": True - }, - "definition": { - "dynamicRegistration": True, - "linkSupport": True - }, - "diagnostic": { - "dynamicRegistration": True, - "relatedDocumentSupport": False - }, - "documentHighlight": { - "dynamicRegistration": True - }, - "documentLink": { - "dynamicRegistration": True, - "tooltipSupport": True + "insertTextMode": 2, }, + "declaration": {"dynamicRegistration": True, "linkSupport": True}, + "definition": {"dynamicRegistration": True, "linkSupport": True}, + "diagnostic": {"dynamicRegistration": True, "relatedDocumentSupport": False}, + "documentHighlight": {"dynamicRegistration": True}, + "documentLink": {"dynamicRegistration": True, "tooltipSupport": True}, "documentSymbol": { "dynamicRegistration": True, "hierarchicalDocumentSymbolSupport": True, "labelSupport": True, "symbolKind": { - "valueSet": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 - ] + "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] }, - "tagSupport": { - "valueSet": [ - 1 - ] - } + "tagSupport": {"valueSet": [1]}, }, "foldingRange": { "dynamicRegistration": True, - "foldingRange": { - "collapsedText": False - }, - "foldingRangeKind": { - "valueSet": [ - "comment", - "imports", - "region" - ] - }, + "foldingRange": {"collapsedText": False}, + "foldingRangeKind": {"valueSet": ["comment", "imports", "region"]}, "lineFoldingOnly": True, - "rangeLimit": 5000 - }, - "formatting": { - "dynamicRegistration": True - }, - "hover": { - "contentFormat": [ - "markdown", - "plaintext" - ], - "dynamicRegistration": True - }, - "implementation": { - "dynamicRegistration": True, - "linkSupport": True + "rangeLimit": 5000, }, + "formatting": {"dynamicRegistration": True}, + "hover": {"contentFormat": ["markdown", "plaintext"], "dynamicRegistration": True}, + "implementation": {"dynamicRegistration": True, "linkSupport": True}, "inlayHint": { "dynamicRegistration": True, - "resolveSupport": { - "properties": [ - "tooltip", - "textEdits", - "label.tooltip", - "label.location", - "label.command" - ] - } - }, - "inlineValue": { - "dynamicRegistration": True - }, - "linkedEditingRange": { - "dynamicRegistration": True - }, - "onTypeFormatting": { - "dynamicRegistration": True + "resolveSupport": {"properties": ["tooltip", "textEdits", "label.tooltip", "label.location", "label.command"]}, }, + "inlineValue": {"dynamicRegistration": True}, + "linkedEditingRange": {"dynamicRegistration": True}, + "onTypeFormatting": {"dynamicRegistration": True}, "publishDiagnostics": { "codeDescriptionSupport": True, "dataSupport": True, "relatedInformation": True, - "tagSupport": { - "valueSet": [ - 1, - 2 - ] - }, - "versionSupport": False - }, - "rangeFormatting": { - "dynamicRegistration": True, - "rangesSupport": True - }, - "references": { - "dynamicRegistration": True + "tagSupport": {"valueSet": [1, 2]}, + "versionSupport": False, }, + "rangeFormatting": {"dynamicRegistration": True, "rangesSupport": True}, + "references": {"dynamicRegistration": True}, "rename": { "dynamicRegistration": True, "honorsChangeAnnotations": True, "prepareSupport": True, - "prepareSupportDefaultBehavior": 1 - }, - "selectionRange": { - "dynamicRegistration": True + "prepareSupportDefaultBehavior": 1, }, + "selectionRange": {"dynamicRegistration": True}, "semanticTokens": { "augmentsSyntaxTokens": True, "dynamicRegistration": True, - "formats": [ - "relative" - ], + "formats": ["relative"], "multilineTokenSupport": False, "overlappingTokenSupport": False, - "requests": { - "full": { - "delta": True - }, - "range": True - }, + "requests": {"full": {"delta": True}, "range": True}, "serverCancelSupport": True, "tokenModifiers": [ "declaration", @@ -348,7 +196,7 @@ class SourceKitLSP(SolidLanguageServer): "async", "modification", "documentation", - "defaultLibrary" + "defaultLibrary", ], "tokenTypes": [ "namespace", @@ -373,67 +221,35 @@ class SourceKitLSP(SolidLanguageServer): "number", "regexp", "operator", - "decorator" - ] + "decorator", + ], }, "signatureHelp": { "contextSupport": True, "dynamicRegistration": True, "signatureInformation": { "activeParameterSupport": True, - "documentationFormat": [ - "markdown", - "plaintext" - ], - "parameterInformation": { - "labelOffsetSupport": True - } - } + "documentationFormat": ["markdown", "plaintext"], + "parameterInformation": {"labelOffsetSupport": True}, + }, }, - "synchronization": { - "didSave": True, - "dynamicRegistration": True, - "willSave": True, - "willSaveWaitUntil": True - }, - "typeDefinition": { - "dynamicRegistration": True, - "linkSupport": True - }, - "typeHierarchy": { - "dynamicRegistration": True - } + "synchronization": {"didSave": True, "dynamicRegistration": True, "willSave": True, "willSaveWaitUntil": True}, + "typeDefinition": {"dynamicRegistration": True, "linkSupport": True}, + "typeHierarchy": {"dynamicRegistration": True}, }, "window": { - "showDocument": { - "support": True - }, - "showMessage": { - "messageActionItem": { - "additionalPropertiesSupport": True - } - }, - "workDoneProgress": True + "showDocument": {"support": True}, + "showMessage": {"messageActionItem": {"additionalPropertiesSupport": True}}, + "workDoneProgress": True, }, "workspace": { "applyEdit": True, - "codeLens": { - "refreshSupport": True - }, + "codeLens": {"refreshSupport": True}, "configuration": True, - "diagnostics": { - "refreshSupport": True - }, - "didChangeConfiguration": { - "dynamicRegistration": True - }, - "didChangeWatchedFiles": { - "dynamicRegistration": True, - "relativePatternSupport": True - }, - "executeCommand": { - "dynamicRegistration": True - }, + "diagnostics": {"refreshSupport": True}, + "didChangeConfiguration": {"dynamicRegistration": True}, + "didChangeWatchedFiles": {"dynamicRegistration": True, "relativePatternSupport": True}, + "executeCommand": {"dynamicRegistration": True}, "fileOperations": { "didCreate": True, "didDelete": True, @@ -441,70 +257,38 @@ class SourceKitLSP(SolidLanguageServer): "dynamicRegistration": True, "willCreate": True, "willDelete": True, - "willRename": True - }, - "foldingRange": { - "refreshSupport": True - }, - "inlayHint": { - "refreshSupport": True - }, - "inlineValue": { - "refreshSupport": True - }, - "semanticTokens": { - "refreshSupport": False + "willRename": True, }, + "foldingRange": {"refreshSupport": True}, + "inlayHint": {"refreshSupport": True}, + "inlineValue": {"refreshSupport": True}, + "semanticTokens": {"refreshSupport": False}, "symbol": { "dynamicRegistration": True, - "resolveSupport": { - "properties": [ - "location.range" - ] - }, + "resolveSupport": {"properties": ["location.range"]}, "symbolKind": { - "valueSet": [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 - ] + "valueSet": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26] }, - "tagSupport": { - "valueSet": [ - 1 - ] - } + "tagSupport": {"valueSet": [1]}, }, "workspaceEdit": { - "changeAnnotationSupport": { - "groupsOnLabel": True - }, + "changeAnnotationSupport": {"groupsOnLabel": True}, "documentChanges": True, "failureHandling": "textOnlyTransactional", "normalizesLineEndings": True, - "resourceOperations": [ - "create", - "rename", - "delete" - ] + "resourceOperations": ["create", "rename", "delete"], }, - "workspaceFolders": True - } - }, - "clientInfo": { - "name": "Visual Studio Code", - "version": "1.102.2" + "workspaceFolders": True, + }, }, + "clientInfo": {"name": "Visual Studio Code", "version": "1.102.2"}, "initializationOptions": { "backgroundIndexing": True, "backgroundPreparationMode": "enabled", - "textDocument/codeLens": { - "supportedCommands": { - "swift.debug": "swift.debug", - "swift.run": "swift.run" - } - }, + "textDocument/codeLens": {"supportedCommands": {"swift.debug": "swift.debug", "swift.run": "swift.run"}}, "window/didChangeActiveDocument": True, "workspace/getReferenceDocument": True, - "workspace/peekDocuments": True + "workspace/peekDocuments": True, }, "locale": "en", "processId": os.getpid(), @@ -517,7 +301,7 @@ class SourceKitLSP(SolidLanguageServer): } ], } - + return initialize_params def _start_server(self): @@ -549,13 +333,18 @@ class SourceKitLSP(SolidLanguageServer): capabilities = init_response["capabilities"] self.logger.log(f"SourceKit LSP capabilities: {list(capabilities.keys())}", logging.INFO) - + assert "textDocumentSync" in capabilities, "textDocumentSync capability missing" assert "definitionProvider" in capabilities, "definitionProvider capability missing" self.server.notify.initialized({}) self.completions_available.set() - # sourcekit-lsp server is typically ready immediately after initialization self.server_ready.set() - self.server_ready.wait() \ No newline at end of file + self.server_ready.wait() + + # SourceKit LSP needs a short initialization period after startup + # before it can provide accurate reference information. This sleep + # prevents race conditions where references might not be available yet. + # Unfortunately, sourcekit doesn't send a signal when it's really ready + time.sleep(5) diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py index 464553e9..67dfe686 100644 --- a/test/solidlsp/swift/test_swift_basic.py +++ b/test/solidlsp/swift/test_swift_basic.py @@ -6,8 +6,6 @@ like request_references using the Swift test repository. """ import os -import sys -import time import platform import pytest @@ -17,30 +15,31 @@ from serena.text_utils import LineType from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language - # Skip Swift tests on Windows due to complex GitHub Actions configuration WINDOWS_SKIP = platform.system() == "Windows" WINDOWS_SKIP_REASON = "GitHub Actions configuration for Swift on Windows is complex, skipping for now." pytestmark = [pytest.mark.swift, pytest.mark.skipif(WINDOWS_SKIP, reason=WINDOWS_SKIP_REASON)] + class TestSwiftLanguageServerBasics: """Test basic functionality of the Swift language server.""" + @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) def test_goto_definition_calculator_class(self, language_server: SolidLanguageServer) -> None: """Test goto_definition on Calculator class usage.""" file_path = os.path.join("src", "main.swift") - + # Find the Calculator usage at line 5: let calculator = Calculator() # Position should be at the "Calculator()" call definitions = language_server.request_definition(file_path, 4, 23) # Position at Calculator() call assert isinstance(definitions, list), "Definitions should be a list" assert len(definitions) > 0, "Should find definition for Calculator class" - + # Verify the definition points to the Calculator class definition calculator_def = definitions[0] assert calculator_def.get("uri", "").endswith("main.swift"), "Definition should be in main.swift" - + # The Calculator class is defined starting at line 16 start_line = calculator_def.get("range", {}).get("start", {}).get("line") assert start_line == 15, f"Calculator class definition should be at line 16, got {start_line + 1}" @@ -49,17 +48,17 @@ class TestSwiftLanguageServerBasics: def test_goto_definition_user_struct(self, language_server: SolidLanguageServer) -> None: """Test goto_definition on User struct usage.""" file_path = os.path.join("src", "main.swift") - + # Find the User usage at line 9: let user = User(name: "Alice", age: 30) # Position should be at the "User(...)" call definitions = language_server.request_definition(file_path, 8, 18) # Position at User(...) call assert isinstance(definitions, list), "Definitions should be a list" assert len(definitions) > 0, "Should find definition for User struct" - + # Verify the definition points to the User struct definition user_def = definitions[0] assert user_def.get("uri", "").endswith("main.swift"), "Definition should be in main.swift" - + # The User struct is defined starting at line 26 start_line = user_def.get("range", {}).get("start", {}).get("line") assert start_line == 25, f"User struct definition should be at line 26, got {start_line + 1}" @@ -68,16 +67,16 @@ class TestSwiftLanguageServerBasics: def test_goto_definition_calculator_method(self, language_server: SolidLanguageServer) -> None: """Test goto_definition on Calculator method usage.""" file_path = os.path.join("src", "main.swift") - + # Find the add method usage at line 6: let result = calculator.add(5, 3) # Position should be at the "add" method call definitions = language_server.request_definition(file_path, 5, 28) # Position at add method call assert isinstance(definitions, list), "Definitions should be a list" - + # Verify the definition points to the add method definition add_def = definitions[0] assert add_def.get("uri", "").endswith("main.swift"), "Definition should be in main.swift" - + # The add method is defined starting at line 17 start_line = add_def.get("range", {}).get("start", {}).get("line") assert start_line == 16, f"add method definition should be at line 17, got {start_line + 1}" @@ -86,42 +85,38 @@ class TestSwiftLanguageServerBasics: def test_goto_definition_cross_file(self, language_server: SolidLanguageServer) -> None: """Test goto_definition across files - Utils struct.""" utils_file = os.path.join("src", "utils.swift") - + # First, let's check if Utils is used anywhere (it might not be in this simple test) # We'll test goto_definition on Utils struct itself symbols = language_server.request_document_symbols(utils_file) utils_symbol = next((s for s in symbols[0] if s.get("name") == "Utils"), None) - + sel_start = utils_symbol["selectionRange"]["start"] definitions = language_server.request_definition(utils_file, sel_start["line"], sel_start["character"]) assert isinstance(definitions, list), "Definitions should be a list" - + # Should find the Utils struct definition itself utils_def = definitions[0] assert utils_def.get("uri", "").endswith("utils.swift"), "Definition should be in utils.swift" @pytest.mark.parametrize("language_server", [Language.SWIFT], indirect=True) def test_request_references_calculator_class(self, language_server: SolidLanguageServer) -> None: - # SourceKit LSP needs a short initialization period after startup - # before it can provide accurate reference information. This sleep - # prevents race conditions where references might not be available yet. - time.sleep(5) """Test request_references on the Calculator class.""" # Get references to the Calculator class in main.swift file_path = os.path.join("src", "main.swift") symbols = language_server.request_document_symbols(file_path) - + calculator_symbol = next((s for s in symbols[0] if s.get("name") == "Calculator"), None) - + sel_start = calculator_symbol["selectionRange"]["start"] references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) assert isinstance(references, list), "References should be a list" assert len(references) > 0, "Calculator class should be referenced" - + # Validate that Calculator is referenced in the main function calculator_refs = [ref for ref in references if ref.get("uri", "").endswith("main.swift")] assert len(calculator_refs) > 0, "Calculator class should be referenced in main.swift" - + # Check that one reference is at line 5 (let calculator = Calculator()) line_5_refs = [ref for ref in calculator_refs if ref.get("range", {}).get("start", {}).get("line") == 4] assert len(line_5_refs) > 0, "Calculator should be referenced at line 5" @@ -132,17 +127,17 @@ class TestSwiftLanguageServerBasics: # Get references to the User struct in main.swift file_path = os.path.join("src", "main.swift") symbols = language_server.request_document_symbols(file_path) - + user_symbol = next((s for s in symbols[0] if s.get("name") == "User"), None) - + sel_start = user_symbol["selectionRange"]["start"] references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) assert isinstance(references, list), "References should be a list" - + # Validate that User is referenced in the main function user_refs = [ref for ref in references if ref.get("uri", "").endswith("main.swift")] assert len(user_refs) > 0, "User struct should be referenced in main.swift" - + # Check that one reference is at line 9 (let user = User(...)) line_9_refs = [ref for ref in user_refs if ref.get("range", {}).get("start", {}).get("line") == 8] assert len(line_9_refs) > 0, "User should be referenced at line 9" @@ -160,17 +155,16 @@ class TestSwiftLanguageServerBasics: references = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) assert isinstance(references, list), "References should be a list" assert len(references) > 0, "Utils struct should be referenced" - + # Validate that Utils is referenced in main.swift utils_refs = [ref for ref in references if ref.get("uri", "").endswith("main.swift")] assert len(utils_refs) > 0, "Utils struct should be referenced in main.swift" - + # Check that one reference is at line 12 (Utils.calculateArea call) line_12_refs = [ref for ref in utils_refs if ref.get("range", {}).get("start", {}).get("line") == 11] assert len(line_12_refs) > 0, "Utils should be referenced at line 12" - class TestSwiftProjectBasics: @pytest.mark.parametrize("project", [Language.SWIFT], indirect=True) def test_retrieve_content_around_line(self, project: Project) -> None: @@ -187,7 +181,7 @@ class TestSwiftProjectBasics: break except: continue - + assert calculator_line is not None, "Calculator class not found" line_calc = project.retrieve_content_around_line(file_path, calculator_line) assert len(line_calc.lines) == 1 @@ -209,7 +203,7 @@ class TestSwiftProjectBasics: user_matches = [m for m in matches if "User" in str(m)] assert len(user_matches) > 0, "Should find User struct" - # Scenario 4: Search for class definitions + # Scenario 4: Search for class definitions class_pattern = r"class\s+\w+" matches = project.search_source_files_for_pattern(class_pattern) assert len(matches) > 0, "Should find class definitions" @@ -225,4 +219,4 @@ class TestSwiftProjectBasics: assert len(matches) > 0, "Should find enum definitions" # Should find Status enum status_matches = [m for m in matches if "Status" in str(m)] - assert len(status_matches) > 0, "Should find Status enum" \ No newline at end of file + assert len(status_matches) > 0, "Should find Status enum" From d41ca01513ded599012de28728c73efe0535dfb5 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 7 Aug 2025 22:21:14 +0200 Subject: [PATCH 057/220] Sleep before requesting references, not after start --- src/solidlsp/language_servers/sourcekit_lsp.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/language_servers/sourcekit_lsp.py b/src/solidlsp/language_servers/sourcekit_lsp.py index f566b693..4a47ffb9 100644 --- a/src/solidlsp/language_servers/sourcekit_lsp.py +++ b/src/solidlsp/language_servers/sourcekit_lsp.py @@ -7,6 +7,7 @@ import time from overrides import override +from solidlsp import ls_types from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger @@ -60,6 +61,7 @@ class SourceKitLSP(SolidLanguageServer): ) self.server_ready = threading.Event() self.request_id = 0 + self._did_sleep_before_requesting_references = False @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: @@ -343,8 +345,14 @@ class SourceKitLSP(SolidLanguageServer): self.server_ready.set() self.server_ready.wait() + @override + def request_references(self, relative_file_path: str, line: int, column: int) -> list[ls_types.Location]: # SourceKit LSP needs a short initialization period after startup # before it can provide accurate reference information. This sleep # prevents race conditions where references might not be available yet. # Unfortunately, sourcekit doesn't send a signal when it's really ready - time.sleep(5) + if not self._did_sleep_before_requesting_references: + self.logger.log("Sleeping 5s before requesting references for the first time", logging.DEBUG) + time.sleep(5) + self._did_sleep_before_requesting_references = True + return super().request_references(relative_file_path, line, column) From b0032ddf9a89c6b09ccc1f3d7f06f9529707eebc Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 7 Aug 2025 22:21:38 +0200 Subject: [PATCH 058/220] Install swift with GH action instead of custom code --- .github/workflows/pytest.yml | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 49d8e857..88424dcb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -70,34 +70,9 @@ jobs: with: terraform_version: "1.5.0" terraform_wrapper: false - - name: Install Swift with swiftly (macOS) - if: runner.os == 'macOS' - run: | - echo "=== Installing swiftly on macOS ===" - curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg && \ - installer -pkg swiftly.pkg -target CurrentUserHomeDirectory && \ - ~/.swiftly/bin/swiftly init --quiet-shell-followup && \ - . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" && \ - hash -r - swiftly install --use 6.1.2 - swiftly use 6.1.2 - echo "~/.swiftly/bin" >> $GITHUB_PATH - echo "Swiftly installed successfully" - - name: Install Swift with swiftly (Ubuntu) - if: runner.os == 'Linux' - run: | - echo "=== Installing swiftly on Ubuntu ===" - curl -O https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz && \ - tar zxf swiftly-$(uname -m).tar.gz && \ - ./swiftly init --quiet-shell-followup && \ - . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ - hash -r - swiftly install --use 6.1.2 - swiftly use 6.1.2 - sudo apt-get -y install libcurl4-openssl-dev - echo "=== Adding Swift toolchain to PATH ===" - echo "$HOME/.local/share/swiftly/bin" >> $GITHUB_PATH - echo "Swiftly installed successfully!" + - name: Install swift + if: runner.os != 'Windows' + uses: swift-actions/setup-swift@v2 - name: Install Ruby uses: ruby/setup-ruby@v1 with: From 80ff428012da7b898ef52cf644bb46bddc91a3f1 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Thu, 7 Aug 2025 22:45:24 +0200 Subject: [PATCH 059/220] Install xtools explictly after swift install action --- .github/workflows/pytest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 88424dcb..04ca150f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -73,6 +73,10 @@ jobs: - name: Install swift if: runner.os != 'Windows' uses: swift-actions/setup-swift@v2 + # Installation of swift screws with installation of ruby on macOS for some reason + - name: Install Xcode Command Line Tools + run: xcode-select --install || true + if: runner.os == 'macOS' - name: Install Ruby uses: ruby/setup-ruby@v1 with: From 4fe88b1817efc56514e8bb889be1eda13166b47d Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Fri, 8 Aug 2025 00:00:21 +0200 Subject: [PATCH 060/220] Revert back to custom code for installing swift --- .github/workflows/pytest.yml | 39 ++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 04ca150f..41c3f186 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -70,13 +70,40 @@ jobs: with: terraform_version: "1.5.0" terraform_wrapper: false - - name: Install swift - if: runner.os != 'Windows' - uses: swift-actions/setup-swift@v2 - # Installation of swift screws with installation of ruby on macOS for some reason - - name: Install Xcode Command Line Tools - run: xcode-select --install || true + # - name: Install swift + # if: runner.os != 'Windows' + # uses: swift-actions/setup-swift@v2 + # Installation of swift with the action screws with installation of ruby on macOS for some reason + # We can try again when version 3 of the action is released, where they will also use swiftly + # Until then, we use custom code to install swift. Sourcekit-lsp is installed automatically with swift + - name: Install Swift with swiftly (macOS) if: runner.os == 'macOS' + run: | + echo "=== Installing swiftly on macOS ===" + curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg && \ + installer -pkg swiftly.pkg -target CurrentUserHomeDirectory && \ + ~/.swiftly/bin/swiftly init --quiet-shell-followup && \ + . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" && \ + hash -r + swiftly install --use 6.1.2 + swiftly use 6.1.2 + echo "~/.swiftly/bin" >> $GITHUB_PATH + echo "Swiftly installed successfully" + - name: Install Swift with swiftly (Ubuntu) + if: runner.os == 'Linux' + run: | + echo "=== Installing swiftly on Ubuntu ===" + curl -O https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz && \ + tar zxf swiftly-$(uname -m).tar.gz && \ + ./swiftly init --quiet-shell-followup && \ + . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ + hash -r + swiftly install --use 6.1.2 + swiftly use 6.1.2 + sudo apt-get -y install libcurl4-openssl-dev + echo "=== Adding Swift toolchain to PATH ===" + echo "$HOME/.local/share/swiftly/bin" >> $GITHUB_PATH + echo "Swiftly installed successfully!" - name: Install Ruby uses: ruby/setup-ruby@v1 with: From f12547d55a219e3adbf1c33512df665b558cf7cd Mon Sep 17 00:00:00 2001 From: Alleo Indong Date: Fri, 8 Aug 2025 06:40:31 +0800 Subject: [PATCH 061/220] Added dark/light mode theme switcher to dashboard (#435) --- src/serena/resources/dashboard/dashboard.js | 203 +++++++++++++++++- src/serena/resources/dashboard/index.html | 185 ++++++++++++---- .../dashboard/serena-logs-dark-mode.png | Bin 0 -> 12994 bytes 3 files changed, 336 insertions(+), 52 deletions(-) create mode 100644 src/serena/resources/dashboard/serena-logs-dark-mode.png diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index 4edc0868..282a3330 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -61,6 +61,9 @@ class Dashboard { this.$statsSection = $('#stats-section'); this.$refreshStats = $('#refresh-stats'); this.$clearStats = $('#clear-stats'); + this.$themeToggle = $('#theme-toggle'); + this.$themeIcon = $('#theme-icon'); + this.$themeText = $('#theme-text'); this.countChart = null; this.tokensChart = null; @@ -73,6 +76,10 @@ class Dashboard { this.$toggleStats.click(this.toggleStats.bind(this)); this.$refreshStats.click(this.loadStats.bind(this)); this.$clearStats.click(this.clearStats.bind(this)); + this.$themeToggle.click(this.toggleTheme.bind(this)); + + // initialize theme + this.initializeTheme(); // initialize the application this.loadToolNames().then(function() { @@ -301,6 +308,11 @@ class Dashboard { // Register datalabels plugin Chart.register(ChartDataLabels); + // Get theme-aware colors + const isDark = document.documentElement.getAttribute('data-theme') === 'dark'; + const textColor = isDark ? '#ffffff' : '#000000'; + const gridColor = isDark ? '#444' : '#ddd'; + // Tool calls pie chart this.countChart = new Chart(countCtx, { type: 'pie', @@ -313,7 +325,12 @@ class Dashboard { }, options: { plugins: { - legend: { display: true }, + legend: { + display: true, + labels: { + color: textColor + } + }, datalabels: { display: true, color: 'white', @@ -336,7 +353,12 @@ class Dashboard { }, options: { plugins: { - legend: { display: true }, + legend: { + display: true, + labels: { + color: textColor + } + }, datalabels: { display: true, color: 'white', @@ -359,7 +381,12 @@ class Dashboard { }, options: { plugins: { - legend: { display: true }, + legend: { + display: true, + labels: { + color: textColor + } + }, datalabels: { display: true, color: 'white', @@ -395,21 +422,56 @@ class Dashboard { }, options: { responsive: true, + plugins: { + legend: { + labels: { + color: textColor + } + } + }, scales: { + x: { + ticks: { + color: textColor + }, + grid: { + color: gridColor + } + }, y: { type: 'linear', display: true, position: 'left', beginAtZero: true, - title: { display: true, text: 'Input Tokens' } + title: { + display: true, + text: 'Input Tokens', + color: textColor + }, + ticks: { + color: textColor + }, + grid: { + color: gridColor + } }, y1: { type: 'linear', display: true, position: 'right', beginAtZero: true, - title: { display: true, text: 'Output Tokens' }, - grid: { drawOnChartArea: false } + title: { + display: true, + text: 'Output Tokens', + color: textColor + }, + ticks: { + color: textColor + }, + grid: { + drawOnChartArea: false, + color: gridColor + } } } } @@ -437,6 +499,135 @@ class Dashboard { $('#stats-summary').html(tableHtml); } + initializeTheme() { + // Check if user has manually set a theme preference + const savedTheme = localStorage.getItem('serena-theme'); + + if (savedTheme) { + // User has manually set a preference, use it + this.setTheme(savedTheme); + } else { + // No manual preference, detect system color scheme + this.detectSystemTheme(); + } + + // Listen for system theme changes + this.setupSystemThemeListener(); + } + + detectSystemTheme() { + // Check if system prefers dark mode + const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches; + const theme = prefersDark ? 'dark' : 'light'; + this.setTheme(theme); + } + + setupSystemThemeListener() { + // Listen for changes in system color scheme + const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)'); + + const handleSystemThemeChange = (e) => { + // Only auto-switch if user hasn't manually set a preference + const savedTheme = localStorage.getItem('serena-theme'); + if (!savedTheme) { + const newTheme = e.matches ? 'dark' : 'light'; + this.setTheme(newTheme); + } + }; + + // Add listener for system theme changes + if (mediaQuery.addEventListener) { + mediaQuery.addEventListener('change', handleSystemThemeChange); + } else { + // Fallback for older browsers + mediaQuery.addListener(handleSystemThemeChange); + } + } + + toggleTheme() { + const currentTheme = document.documentElement.getAttribute('data-theme') || 'light'; + const newTheme = currentTheme === 'light' ? 'dark' : 'light'; + + // When user manually toggles, save their preference + localStorage.setItem('serena-theme', newTheme); + this.setTheme(newTheme); + } + + setTheme(theme) { + // Set the theme on the document element + document.documentElement.setAttribute('data-theme', theme); + + // Update the toggle button + if (theme === 'dark') { + this.$themeIcon.text('☀️'); + this.$themeText.text('Light'); + } else { + this.$themeIcon.text('🌙'); + this.$themeText.text('Dark'); + } + + // Update the logo based on theme + this.updateLogo(theme); + + // Save to localStorage + localStorage.setItem('serena-theme', theme); + + // Update charts if they exist + this.updateChartsTheme(); + } + + updateLogo(theme) { + const logoElement = document.getElementById('serena-logo'); + if (logoElement) { + if (theme === 'dark') { + logoElement.src = 'serena-logs-dark-mode.png'; + } else { + logoElement.src = 'serena-logs.png'; + } + } + } + + updateChartsTheme() { + const isDark = document.documentElement.getAttribute('data-theme') === 'dark'; + const textColor = isDark ? '#ffffff' : '#000000'; + const gridColor = isDark ? '#444' : '#ddd'; + + // Update existing charts + if (this.countChart) { + this.countChart.options.scales.x.ticks.color = textColor; + this.countChart.options.scales.y.ticks.color = textColor; + this.countChart.options.scales.x.grid.color = gridColor; + this.countChart.options.scales.y.grid.color = gridColor; + this.countChart.update(); + } + + if (this.inputChart) { + this.inputChart.options.scales.x.ticks.color = textColor; + this.inputChart.options.scales.y.ticks.color = textColor; + this.inputChart.options.scales.x.grid.color = gridColor; + this.inputChart.options.scales.y.grid.color = gridColor; + this.inputChart.update(); + } + + if (this.outputChart) { + this.outputChart.options.scales.x.ticks.color = textColor; + this.outputChart.options.scales.y.ticks.color = textColor; + this.outputChart.options.scales.x.grid.color = gridColor; + this.outputChart.options.scales.y.grid.color = gridColor; + this.outputChart.update(); + } + + if (this.tokensChart) { + this.tokensChart.options.scales.x.ticks.color = textColor; + this.tokensChart.options.scales.y.ticks.color = textColor; + this.tokensChart.options.scales.y1.ticks.color = textColor; + this.tokensChart.options.scales.x.grid.color = gridColor; + this.tokensChart.options.scales.y.grid.color = gridColor; + this.tokensChart.options.scales.y1.grid.color = gridColor; + this.tokensChart.update(); + } + } + shutdown() { const self = this; const _shutdown = function () { diff --git a/src/serena/resources/dashboard/index.html b/src/serena/resources/dashboard/index.html index a7d21c96..a303839a 100644 --- a/src/serena/resources/dashboard/index.html +++ b/src/serena/resources/dashboard/index.html @@ -1,5 +1,6 @@ + @@ -12,21 +13,65 @@ +
- Serena +
- +
+
+ 🌙 + Dark +
@@ -184,14 +276,14 @@
- +
-