mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 16:17:25 +00:00
BuXiuDN
9c74826cb4
Add skills/anysearch (vendored from anysearch-ai/anysearch-skill v3.1.1) so agents get general/anonymous search, parallel batch search, vertical domain search, and full-page extract. Includes MAINTENANCE.md and docs/anysearch-skill.md. No internal/, ui/, or migration changes. Disclosure: AnySearch Open Source Bounty Claim goclaw#001.
244 lines
9.3 KiB
Python
244 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Code generator for AnySearch CLI scripts.
|
|
|
|
Reads the skill version from SKILL.md and shared data from scripts/shared/, then
|
|
injects the client header, API base URL, domain list, and doc command
|
|
implementation into each CLI script. Eliminates duplication across all 4
|
|
language implementations.
|
|
|
|
Usage:
|
|
python scripts/generate.py # Generate all scripts
|
|
python scripts/generate.py --check # Verify scripts are up-to-date (for CI)
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
SHARED_DIR = os.path.join(SCRIPT_DIR, "shared")
|
|
SKILL_PATH = os.path.join(os.path.dirname(SCRIPT_DIR), "SKILL.md")
|
|
|
|
# --- Marker format per language ---
|
|
# Each script uses paired comments to delimit generated sections:
|
|
# BEGIN GENERATED:<section_name>
|
|
# ... generated content ...
|
|
# END GENERATED:<section_name>
|
|
|
|
MARKERS = {
|
|
".py": ("# BEGIN GENERATED:{name}", "# END GENERATED:{name}"),
|
|
".js": ("// BEGIN GENERATED:{name}", "// END GENERATED:{name}"),
|
|
".ps1": ("# BEGIN GENERATED:{name}", "# END GENERATED:{name}"),
|
|
".sh": ("# BEGIN GENERATED:{name}", "# END GENERATED:{name}"),
|
|
}
|
|
|
|
|
|
def load_constants():
|
|
with open(os.path.join(SHARED_DIR, "constants.json"), "r", encoding="utf-8") as f:
|
|
constants = json.load(f)
|
|
constants["skill_version"] = load_skill_version()
|
|
return constants
|
|
|
|
|
|
def load_skill_version():
|
|
"""Read and validate the version from the SKILL.md frontmatter."""
|
|
with open(SKILL_PATH, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
parts = content.split("---", 2)
|
|
if len(parts) < 3 or parts[0].strip():
|
|
raise ValueError("SKILL.md must start with YAML frontmatter")
|
|
|
|
matches = re.findall(r"^version:\s*([^\s#]+)\s*(?:#.*)?$", parts[1], re.MULTILINE)
|
|
if len(matches) != 1:
|
|
raise ValueError("SKILL.md frontmatter must contain exactly one version")
|
|
|
|
version = matches[0]
|
|
semver = r"\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?"
|
|
if not re.fullmatch(semver, version):
|
|
raise ValueError(f"Invalid SKILL.md version: {version}")
|
|
return version
|
|
|
|
|
|
def render_constants(ext, constants):
|
|
"""Render constants block in the target language syntax."""
|
|
client_header = f'skill/{constants["skill_version"]}'
|
|
base_url = constants["api_base_url"]
|
|
domains = constants["available_domains"]
|
|
|
|
if ext == ".py":
|
|
lines = [f'CLIENT_HEADER = "{client_header}"']
|
|
lines.append(f'API_BASE_URL = os.environ.get("ANYSEARCH_API_BASE_URL", "{base_url}").rstrip("/")')
|
|
lines.append("AVAILABLE_DOMAINS = [")
|
|
for i in range(0, len(domains), 6):
|
|
chunk = domains[i:i+6]
|
|
lines.append(" " + ", ".join(f'"{d}"' for d in chunk) + ",")
|
|
lines.append("]")
|
|
return "\n".join(lines)
|
|
|
|
elif ext == ".js":
|
|
lines = [f'const CLIENT_HEADER = "{client_header}";']
|
|
lines.append(f'const API_BASE_URL = (process.env.ANYSEARCH_API_BASE_URL || "{base_url}").replace(/\\/$/, "");')
|
|
lines.append("const AVAILABLE_DOMAINS = [")
|
|
for i in range(0, len(domains), 6):
|
|
chunk = domains[i:i+6]
|
|
lines.append(" " + ",".join(f'"{d}"' for d in chunk) + ",")
|
|
lines.append("];")
|
|
return "\n".join(lines)
|
|
|
|
elif ext == ".ps1":
|
|
lines = [f'$CLIENT_HEADER = "{client_header}"']
|
|
lines.append(f'$API_BASE_URL = if ($env:ANYSEARCH_API_BASE_URL) {{ $env:ANYSEARCH_API_BASE_URL.TrimEnd("/") }} else {{ "{base_url}" }}')
|
|
lines.append("$AVAILABLE_DOMAINS = @(")
|
|
chunks = [domains[i:i+6] for i in range(0, len(domains), 6)]
|
|
for idx, chunk in enumerate(chunks):
|
|
suffix = "," if idx < len(chunks) - 1 else ""
|
|
lines.append(" " + ", ".join(f'"{d}"' for d in chunk) + suffix)
|
|
lines.append(")")
|
|
return "\n".join(lines)
|
|
|
|
elif ext == ".sh":
|
|
lines = [f'CLIENT_HEADER="{client_header}"']
|
|
lines.extend([f'API_BASE_URL="${{ANYSEARCH_API_BASE_URL:-{base_url}}}"', 'API_BASE_URL="${API_BASE_URL%/}"'])
|
|
lines.append("AVAILABLE_DOMAINS=(" + " ".join(f'"{d}"' for d in domains) + ")")
|
|
return "\n".join(lines)
|
|
|
|
raise ValueError(f"Unsupported extension: {ext}")
|
|
|
|
|
|
def render_doc_block(ext, constants):
|
|
"""Generate code that reads and renders doc_spec.md at runtime."""
|
|
if ext == ".py":
|
|
return '''def _render_doc():
|
|
import json as _json
|
|
_dir = os.path.dirname(os.path.abspath(__file__))
|
|
_shared = os.path.join(_dir, "shared")
|
|
with open(os.path.join(_shared, "doc_spec.md"), "r", encoding="utf-8") as _f:
|
|
_tpl = _f.read()
|
|
with open(os.path.join(_shared, "constants.json"), "r", encoding="utf-8") as _f:
|
|
_c = _json.load(_f)
|
|
_tpl = _tpl.replace("{{LANG_NAME}}", "Python")
|
|
_tpl = _tpl.replace("{{LANG_CODEBLOCK}}", "")
|
|
_tpl = _tpl.replace("{{LANG_INVOKE}}", "python scripts/anysearch_cli.py")
|
|
_tpl = _tpl.replace("{{DOMAINS_SPACE}}", " ".join(_c["available_domains"]))
|
|
return _tpl'''
|
|
|
|
elif ext == ".js":
|
|
return '''function renderDoc() {
|
|
const shared = path.join(__dirname, "shared");
|
|
let tpl = fs.readFileSync(path.join(shared, "doc_spec.md"), "utf-8");
|
|
const c = JSON.parse(fs.readFileSync(path.join(shared, "constants.json"), "utf-8"));
|
|
tpl = tpl.replace(/\\{\\{LANG_NAME\\}\\}/g, "Node.js");
|
|
tpl = tpl.replace(/\\{\\{LANG_CODEBLOCK\\}\\}/g, "");
|
|
tpl = tpl.replace(/\\{\\{LANG_INVOKE\\}\\}/g, "node scripts/anysearch_cli.js");
|
|
tpl = tpl.replace(/\\{\\{DOMAINS_SPACE\\}\\}/g, c.available_domains.join(" "));
|
|
return tpl;
|
|
}'''
|
|
|
|
elif ext == ".ps1":
|
|
return '''function Render-Doc {
|
|
$shared = Join-Path (Split-Path -Parent $MyInvocation.ScriptName) "shared"
|
|
$tpl = Get-Content (Join-Path $shared "doc_spec.md") -Raw -Encoding UTF8
|
|
$c = Get-Content (Join-Path $shared "constants.json") -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
$tpl = $tpl.Replace("{{LANG_NAME}}", "PowerShell")
|
|
$tpl = $tpl.Replace("{{LANG_CODEBLOCK}}", "powershell")
|
|
$tpl = $tpl.Replace("{{LANG_INVOKE}}", "powershell -ExecutionPolicy Bypass -File scripts/anysearch_cli.ps1")
|
|
$tpl = $tpl.Replace("{{DOMAINS_SPACE}}", ($c.available_domains -join " "))
|
|
return $tpl
|
|
}'''
|
|
|
|
elif ext == ".sh":
|
|
return r'''_cmd_doc() {
|
|
local shared="$SCRIPT_DIR/shared"
|
|
local tpl
|
|
tpl=$(cat "$shared/doc_spec.md")
|
|
local domains
|
|
domains=$(jq -r '.available_domains | join(" ")' "$shared/constants.json")
|
|
tpl="${tpl//\{\{LANG_NAME\}\}/Bash}"
|
|
tpl="${tpl//\{\{LANG_CODEBLOCK\}\}/bash}"
|
|
tpl="${tpl//\{\{LANG_INVOKE\}\}/bash scripts/anysearch_cli.sh}"
|
|
tpl="${tpl//\{\{DOMAINS_SPACE\}\}/$domains}"
|
|
printf '%s\n' "$tpl"
|
|
}'''
|
|
|
|
raise ValueError(f"Unsupported extension: {ext}")
|
|
|
|
|
|
def replace_marker_section(content, ext, section_name, new_text):
|
|
"""Replace everything between marker comments for section_name with new_text."""
|
|
begin_tag, end_tag = MARKERS[ext]
|
|
begin = begin_tag.format(name=section_name)
|
|
end = end_tag.format(name=section_name)
|
|
|
|
if begin not in content:
|
|
raise ValueError(f"BEGIN marker '{begin_tag.format(name=section_name)}' not found")
|
|
if end not in content:
|
|
raise ValueError(f"END marker '{end_tag.format(name=section_name)}' not found")
|
|
|
|
before, rest = content.split(begin, 1)
|
|
_, after = rest.split(end, 1)
|
|
return before + begin + "\n" + new_text + "\n" + end + after
|
|
|
|
|
|
def generate_script(script_path, constants):
|
|
"""Regenerate the constants and doc blocks in a CLI script."""
|
|
ext = os.path.splitext(script_path)[1]
|
|
if ext not in MARKERS:
|
|
raise ValueError(f"Unsupported extension: {ext}")
|
|
|
|
with open(script_path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
constants_text = render_constants(ext, constants)
|
|
content = replace_marker_section(content, ext, "CONSTANTS", constants_text)
|
|
|
|
doc_block = render_doc_block(ext, constants)
|
|
content = replace_marker_section(content, ext, "DOC_SPEC", doc_block)
|
|
|
|
return content
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Generate AnySearch CLI scripts from shared data")
|
|
parser.add_argument("--check", action="store_true", help="Verify scripts are up-to-date (for CI)")
|
|
args = parser.parse_args()
|
|
|
|
constants = load_constants()
|
|
|
|
scripts_changed = False
|
|
|
|
for ext in [".py", ".js", ".ps1", ".sh"]:
|
|
script_name = f"anysearch_cli{ext}"
|
|
script_path = os.path.join(SCRIPT_DIR, script_name)
|
|
|
|
try:
|
|
new_content = generate_script(script_path, constants)
|
|
with open(script_path, "r", encoding="utf-8") as f:
|
|
old_content = f.read()
|
|
|
|
if new_content != old_content:
|
|
scripts_changed = True
|
|
if not args.check:
|
|
# Keep Bash runnable from Windows worktrees with core.autocrlf=true.
|
|
# The repository also pins *.sh to LF in .gitattributes.
|
|
with open(script_path, "w", encoding="utf-8", newline="\n") as f:
|
|
f.write(new_content)
|
|
print(f"Generated: {script_name}")
|
|
else:
|
|
print(f"CHANGED: {script_name} (run generate.py to update)")
|
|
else:
|
|
print(f"OK: {script_name}")
|
|
except Exception as e:
|
|
print(f"ERROR in {script_name}: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
if args.check and scripts_changed:
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|