diff --git a/.github/workflows/docs-audit.yml b/.github/workflows/docs-audit.yml index 7c818c5a..b8a9880f 100644 --- a/.github/workflows/docs-audit.yml +++ b/.github/workflows/docs-audit.yml @@ -2,8 +2,6 @@ name: Docs Audit on: pull_request: - push: - branches: [main, master] workflow_dispatch: permissions: @@ -18,9 +16,6 @@ jobs: with: node-version: '24' cache: npm - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - run: npm ci - run: npx playwright install --with-deps chromium - run: npm run check diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 699f94ca..98fdaea3 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -21,13 +21,19 @@ jobs: with: node-version: '24' cache: npm - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - run: npm ci # Upstream ingestion is manual. Validate only the committed public sources. - run: npx playwright install --with-deps chromium - run: npm run check + - name: Verify source files remain unchanged + run: | + git diff --check + git diff --exit-code + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: browser-test-results + path: test-results/ - uses: actions/upload-pages-artifact@v3 with: path: public diff --git a/README.md b/README.md index 90440daf..b712dd4f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ chapters, and documentation for Runtime, Silk, Virtnosis, Sage, and slg. ## Local development -Use Node 24 and Python 3.12. No sibling checkout is needed to build the site. +Use Node 24. Builds, ingestion tools, audits, and tests run on Node; Python is +not required. No sibling checkout is needed to build the site. ```sh npm ci @@ -52,8 +53,8 @@ checkouts, preserve the established curated/website-owned content, and write committed DOMStack Markdown pages. Review and commit their changes before deploying. ```sh -python3 silk/tools/sync-from-silk-docs.py --silk-repo /path/to/silk -python3 runtime/tools/generate-js-api-reference.py --runtime-repo /path/to/runtime +node silk/tools/sync-from-silk-docs.ts --silk-repo /path/to/silk +node runtime/tools/generate-js-api-reference.ts --runtime-repo /path/to/runtime npm run build npm run audit npm run audit:content @@ -69,8 +70,11 @@ linked API headings become plain-text titles when a page changes. The reference catalog includes new pages in the same import batch. Runtime's generated reference markers retain surrounding prose. -The Python index and LLM exporters have been replaced by DOMStack templates. -Content audits inspect `public/` by default; set `ORO_SITE_OUTPUT` to inspect +The generators call the TypeScript importer directly; they never run during a +DOMStack build. Search indexes and LLM exports remain DOMStack templates. +Content audits derive document IDs and expected raw filenames from committed +Markdown frontmatter rather than public navigation JSON, which is still generated. +They inspect `public/` by default; set `ORO_SITE_OUTPUT` to inspect another output directory. Set `ORO_RUNTIME_REPO` explicitly to additionally audit against a particular upstream Runtime checkout; ordinary checks are independent of whatever happens to be checked out next door. @@ -117,7 +121,10 @@ update the relevant assertions and redirects together. ## Deployment and dependency updates -GitHub Actions builds and uploads `public/` and deploys with the Pages environment. +Pull requests run the Docs Audit workflow. Production-branch pushes run the +Pages workflow, which validates once, uploads that checked `public/` artifact, +and deploys with the Pages environment. Both workflows use Node only and retain +browser failure artifacts. The repository's **Settings → Pages → Source** must be **GitHub Actions**. Merging these changes does not itself change that repository setting. The output contains `CNAME`, `.nojekyll`, branding assets, raw Markdown, and `llms.txt` packs. diff --git a/docs/migration.md b/docs/migration.md index 08b0bfb4..6498babb 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -14,9 +14,9 @@ remains reproducible. Production cutover is complete; deployment evidence is rec | Page chrome | Headers, footers, metadata, product navigation, learn chapter bars, documentation sidebars, breadcrumbs, previous/next links, and heading lists render at build time. | | Progressive features | Search, keyboard tabs, copy controls, Ask AI, mobile sidebar controls, active headings, and tab fragments work with browser JavaScript. | | Artifacts | One build emits HTML, indexes, search data, raw Markdown, LLM packs, the sitemap, CNAME, and `.nojekyll`. | -| Ingestion | Silk and Runtime retain their upstream/editorial ownership rules and accept explicit upstream checkout paths. Unchanged public pages retain exact Markdown and metadata. | +| Ingestion | Native TypeScript tools for Silk and Runtime retain their upstream/editorial ownership rules and accept explicit upstream checkout paths. Unchanged public pages retain exact Markdown and metadata. | | Cleanup | The browser Markdown renderers, vendored rendering libraries, old HTML shells, duplicate Python exporters, generated source-tree indexes, and Jekyll configuration are retired. | -| CI | Audit and deployment workflows use Node 24, Python 3.12, and the same `npm run check` gate. | +| CI | Node 24 runs ingestion, audits, browser serving, and the `npm run check` gate without Python. Pull requests validate in Docs Audit; production pushes validate and deploy the same artifact in Pages, without a duplicate push audit. | The route inventory is in `tools/migration/baseline.json`; original Markdown links and heading text are in `tools/migration/link-inventory.json`. The migration diff --git a/package.json b/package.json index 0f39c367..f7a78e21 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "test:site": "node --test tests/site/*.test.ts", "test:browser": "playwright test", "test:reproducibility": "node tools/reproducibility.ts", - "audit:content": "python3 runtime/tools/audit-site.py && python3 silk/tools/audit-site.py && python3 silk/tools/audit-stdlib-docs.py && python3 sage/tools/audit-site.py && python3 slg/tools/audit-site.py && python3 virtnosis/tools/audit-site.py", + "audit:content": "node tools/audit-content.ts", "test:tooling": "node --test tests/tooling/*.test.ts", "check": "npm run typecheck && npm test && npm run build && npm run audit && npm run audit:content && npm run test:site && npm run test:browser && npm run test:tooling && npm run test:reproducibility" }, @@ -32,5 +32,8 @@ "markdown-it": "^15.0.0", "markdown-it-github-alerts": "^1.0.1", "typescript": "^5.9.2" + }, + "imports": { + "#lib/*": "./src/lib/*" } } diff --git a/playwright.config.ts b/playwright.config.ts index 28713868..ab6dddd3 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -9,7 +9,7 @@ export default defineConfig({ : {}, }, webServer: { - command: 'python3 -m http.server 9877 --bind 127.0.0.1 --directory public', + command: 'node tools/serve-output.ts', url: 'http://127.0.0.1:9877', reuseExistingServer: !process.env.CI, }, diff --git a/runtime/tools/audit-site.py b/runtime/tools/audit-site.py deleted file mode 100644 index c183de2c..00000000 --- a/runtime/tools/audit-site.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import re -import sys -import os -from pathlib import Path - - -SITE_ROOT = Path(__file__).resolve().parents[2] -RUNTIME_ROOT = Path(os.environ.get("ORO_SITE_OUTPUT", str(SITE_ROOT / "public"))) / "runtime" -REPO_ROOT = RUNTIME_ROOT.parent -JAVASCRIPT_DOCS = RUNTIME_ROOT / "docs" / "source" / "javascript" -CLI_DOCS = RUNTIME_ROOT / "docs" / "source" / "cli" -UPSTREAM_RUNTIME_ROOT = Path(os.environ["ORO_RUNTIME_REPO"]) if "ORO_RUNTIME_REPO" in os.environ else None -UPSTREAM_RUNTIME_INDEX = UPSTREAM_RUNTIME_ROOT / "api" / "index.d.ts" if UPSTREAM_RUNTIME_ROOT else None -UPSTREAM_RUNTIME_CLI = UPSTREAM_RUNTIME_ROOT / "api" / "CLI.md" if UPSTREAM_RUNTIME_ROOT else None - -sys.path.insert(0, str(SITE_ROOT / "tools")) - -from site_audit_common import SiteAuditConfig, run_site_audit - - -CURATED_FILES_BY_FAMILY = { - "oro:ai": "ai.md", - "oro:application": "application.md", - "oro:extension": "extension.md", - "oro:fs": "fs.md", - "oro:hooks": "hooks.md", - "oro:mcp": "mcp.md", - "oro:notification": "notification.md", - "oro:secure-storage": "secure-storage.md", - "oro:window": "window.md", -} - - -EXCLUDED_PUBLIC_FAMILIES = { - "oro:bootstrap", - "oro:external", - "oro:internal", - "oro:node", - "oro:node-esm-loader", -} - - -MODULE_START = re.compile(r"^declare module ['\"](?Poro:[^'\"]+)['\"]", re.M) -EXAMPLE_HEADING_RE = re.compile(r"^\s*##\s+.*examples?\b", re.I | re.M) - -CLI_SECTION_TO_DOC = { - "oroc": "cli/oroc.md", - "oroc help": "cli/help.md", - "oroc update": "cli/update.md", - "oroc update-init": "cli/update/init.md", - "oroc update-server": "cli/update/server.md", - "oroc update-info": "cli/update/info.md", - "oroc update-keygen": "cli/update/keygen.md", - "oroc update-sign": "cli/update/sign.md", - "oroc update-verify": "cli/update/verify.md", - "oroc update-validate": "cli/update/validate.md", - "oroc update-bundle": "cli/update/bundle.md", - "oroc update-extract": "cli/update/extract.md", - "oroc build": "cli/build.md", - "oroc run": "cli/run.md", - "oroc list-devices": "cli/list-devices.md", - "oroc env": "cli/env.md", - "oroc mcp": "cli/mcp.md", - "oroc init": "cli/init.md", - "oroc install-app": "cli/install-app.md", - "oroc print-build-dir": "cli/print-build-dir.md", - "oroc setup": "cli/setup.md", - "oroc config": "cli/config.md", - "oroc versions": "cli/versions.md", - "oroc version": "cli/version.md", -} - - -def family_reference_re(family: str) -> re.Pattern[str]: - return re.compile(rf"`?{re.escape(family)}(?:/\*|/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)`?") - - -def parse_runtime_specifiers() -> tuple[set[str], set[str]]: - if UPSTREAM_RUNTIME_INDEX is None or not UPSTREAM_RUNTIME_INDEX.exists(): - return set(), set() - text = UPSTREAM_RUNTIME_INDEX.read_text(encoding="utf-8") - specifiers = set(MODULE_START.findall(text)) - families = {spec.split("/", 1)[0] for spec in specifiers} - return families, specifiers - - -def parse_runtime_cli_sections() -> set[str]: - if UPSTREAM_RUNTIME_CLI is None or not UPSTREAM_RUNTIME_CLI.exists(): - return set() - text = UPSTREAM_RUNTIME_CLI.read_text(encoding="utf-8") - return set(re.findall(r"^##\s+([^\n]+)$", text, re.M)) - - -def iter_runtime_doc_issues() -> list[str]: - issues: list[str] = [] - - if (JAVASCRIPT_DOCS / "module-index.md").exists(): - issues.append( - f"{(JAVASCRIPT_DOCS / 'module-index.md').relative_to(REPO_ROOT)}: remove this page; link JavaScript API entry points to javascript/overview." - ) - - for path in sorted(JAVASCRIPT_DOCS.glob("*.md")): - text = path.read_text(encoding="utf-8") - rel = path.relative_to(REPO_ROOT) - - if "console.log(Object.keys(api))" in text: - issues.append(f"{rel}: remove generic Object.keys(api) examples; document a real usage flow instead.") - - if ( - path.stem not in {"overview", "all-modules"} - and path.name not in set(CURATED_FILES_BY_FAMILY.values()) - and not EXAMPLE_HEADING_RE.search(text) - ): - issues.append(f"{rel}: missing an Examples section.") - - families, specifiers = parse_runtime_specifiers() - if families and specifiers: - public_families = {family for family in families if family not in EXCLUDED_PUBLIC_FAMILIES} - public_specifiers = { - spec for spec in specifiers if spec.split("/", 1)[0] not in EXCLUDED_PUBLIC_FAMILIES - } - - for family in sorted(public_families): - expected = CURATED_FILES_BY_FAMILY.get(family, f"{family.removeprefix('oro:')}.md") - if not (JAVASCRIPT_DOCS / expected).exists(): - issues.append( - f"{(JAVASCRIPT_DOCS / expected).relative_to(REPO_ROOT)}: missing docs page for published module family {family}." - ) - - for family in sorted(EXCLUDED_PUBLIC_FAMILIES): - excluded_path = JAVASCRIPT_DOCS / f"{family.removeprefix('oro:')}.md" - if excluded_path.exists(): - issues.append( - f"{excluded_path.relative_to(REPO_ROOT)}: excluded private module family {family} should not have a public docs page." - ) - - all_modules = (JAVASCRIPT_DOCS / "all-modules.md").read_text(encoding="utf-8") - - for family in sorted(EXCLUDED_PUBLIC_FAMILIES): - family_ref = family_reference_re(family) - if family_ref.search(all_modules): - issues.append( - f"{(JAVASCRIPT_DOCS / 'all-modules.md').relative_to(REPO_ROOT)}: excluded private module family {family} should not appear in the public module listing." - ) - - for spec in sorted(public_specifiers): - if spec not in all_modules: - issues.append( - f"{(JAVASCRIPT_DOCS / 'all-modules.md').relative_to(REPO_ROOT)}: missing published module specifier {spec}." - ) - - cli_sections = parse_runtime_cli_sections() - if cli_sections: - for section, rel_path in sorted(CLI_SECTION_TO_DOC.items()): - if section not in cli_sections: - continue - path = RUNTIME_ROOT / "docs" / "source" / rel_path - if not path.exists(): - issues.append( - f"{path.relative_to(REPO_ROOT)}: missing website docs page for upstream CLI section `{section}`." - ) - - stale_repo_patterns = ( - (re.compile(r"oro-computer/" + r"oro" + r"-runtime"), "deprecated runtime repository alias"), - (re.compile(r"oro-computer/" + r"legacy" + r"-runtime"), "deprecated runtime repository alias"), - (re.compile(r"\blegacy" + r"-runtime\b"), "deprecated runtime repository alias"), - ) - for path in ( - REPO_ROOT / "index.html", - RUNTIME_ROOT / "index.html", - RUNTIME_ROOT / "docs" / "index.html", - ): - text = path.read_text(encoding="utf-8") - for pattern, label in stale_repo_patterns: - if pattern.search(text): - issues.append(f"{path.relative_to(REPO_ROOT)}: stale runtime repository reference ({label}).") - - return issues - - -def main() -> int: - rc = run_site_audit( - SiteAuditConfig( - project_root=RUNTIME_ROOT, - docs_source=RUNTIME_ROOT / "docs" / "source", - docs_index=RUNTIME_ROOT / "docs" / "index.json", - missing_index_message="Missing Runtime docs/index.json; run build scripts first.", - success_message="OK: runtime site audit passed", - ) - ) - if rc != 0: - return rc - - issues = iter_runtime_doc_issues() - if issues: - for issue in issues[:200]: - print(issue, file=sys.stderr) - if len(issues) > 200: - print(f"... and {len(issues) - 200} more", file=sys.stderr) - print(f"FAIL: {len(issues)} runtime docs issues", file=sys.stderr) - return 1 - - print("OK: runtime docs audit passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/runtime/tools/generate-js-api-reference.py b/runtime/tools/generate-js-api-reference.py deleted file mode 100644 index 726f7e73..00000000 --- a/runtime/tools/generate-js-api-reference.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import re -import sys -import tempfile -from dataclasses import dataclass -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) -from importlib import import_module -source_pages = import_module("source-pages") - -from js_api_reference_content import ( - DEFAULT_SEE_ALSO, - DESCRIPTION_BY_FAMILY, - EXAMPLES_BY_FAMILY, - GUIDE_REFS_BY_FAMILY, -) - - -MODULE_START = re.compile(r"^declare module ['\"](?Poro:[^'\"]+)['\"]\s*\{\s*$") - - -CURATED_FAMILIES = { - "oro:ai", - "oro:application", - "oro:extension", - "oro:fs", - "oro:hooks", - "oro:mcp", - "oro:notification", - "oro:secure-storage", - "oro:window", -} - - -EXCLUDED_PUBLIC_FAMILIES = { - "oro:bootstrap", - "oro:external", - "oro:internal", - "oro:node", - "oro:node-esm-loader", -} - - -CURATED_FILES_BY_FAMILY = { - "oro:ai": "ai.md", - "oro:application": "application.md", - "oro:extension": "extension.md", - "oro:fs": "fs.md", - "oro:hooks": "hooks.md", - "oro:mcp": "mcp.md", - "oro:notification": "notification.md", - "oro:secure-storage": "secure-storage.md", - "oro:window": "window.md", -} - - -REF_START = "" -REF_END = "" - - -@dataclass(frozen=True) -class ModuleBlock: - spec: str - block: str - - -def repo_root() -> Path: - return Path(__file__).resolve().parents[2] - - -def read_text(path: Path) -> str: - return path.read_text(encoding="utf-8") - - -def write_text_if_changed(path: Path, text: str) -> bool: - path.parent.mkdir(parents=True, exist_ok=True) - prev = path.read_text(encoding="utf-8") if path.exists() else None - if prev == text: - return False - path.write_text(text, encoding="utf-8") - return True - - -def parse_index_d_ts(text: str) -> dict[str, ModuleBlock]: - lines = text.splitlines() - out: dict[str, ModuleBlock] = {} - i = 0 - - while i < len(lines): - m = MODULE_START.match(lines[i]) - if not m: - i += 1 - continue - - spec = m.group("name").strip() - buf: list[str] = [lines[i].rstrip()] - i += 1 - - while i < len(lines): - line = lines[i].rstrip() - buf.append(line) - if line == "}" and not lines[i].startswith((" ", "\t")): - break - i += 1 - - block = "\n".join(buf).rstrip() + "\n" - out[spec] = ModuleBlock(spec=spec, block=block) - i += 1 - - if not out: - raise SystemExit("No `declare module 'oro:*' { ... }` blocks found.") - - return out - - -def family_of(spec: str) -> str: - return spec.split("/", 1)[0] - - -def sort_specs_in_family(family: str, specs: list[str]) -> list[str]: - return sorted(specs, key=lambda s: (s != family, s)) - - -def preferred_import_spec(family: str, specs: list[str], blocks: dict[str, ModuleBlock]) -> str: - if family in blocks: - return family - for s in specs: - if s.endswith("/index"): - return s - return specs[0] - - -def family_title(family: str, blocks: dict[str, ModuleBlock]) -> str: - if family in blocks: - return f"`{family}`" - return f"`{family}/*`" - - -def family_intro(family: str) -> str: - return DESCRIPTION_BY_FAMILY.get( - family, - f"This page documents the exported JavaScript surface for `{family}` as declared by the runtime’s published TypeScript definitions.\n", - ) - - -def render_examples_section(family: str) -> str: - body = EXAMPLES_BY_FAMILY.get(family) - if not body: - raise SystemExit(f"Missing generated examples for {family}") - return f"## Examples\n\n{body.rstrip()}\n" - - -def render_see_also_section() -> str: - lines = ["## See also", ""] - for label, doc_id in DEFAULT_SEE_ALSO: - lines.append(f"- [{label}](?p={doc_id})") - lines.append("") - return "\n".join(lines) - - -def render_related_guides_section(family: str) -> str: - refs = GUIDE_REFS_BY_FAMILY.get(family, ()) - if not refs: - return "" - - lines = ["## Related guides", ""] - for label, doc_id in refs: - lines.append(f"- [{label}](?p={doc_id})") - lines.append("") - return "\n".join(lines) - - -def render_reference_section(family: str, specs: list[str], blocks: dict[str, ModuleBlock]) -> str: - specs_sorted = sort_specs_in_family(family, specs) - - lines: list[str] = [] - lines.append("## API reference") - lines.append("") - lines.append(REF_START) - lines.append("") - lines.append("### Module specifiers") - lines.append("") - lines.append("```text") - for s in specs_sorted: - lines.append(s) - lines.append("```") - lines.append("") - - lines.append("### TypeScript declarations") - lines.append("") - lines.append("These declarations are generated from the runtime's published TypeScript surface.") - lines.append("") - for s in specs_sorted: - lines.append(f"#### `{s}`") - lines.append("") - lines.append("```ts") - lines.extend(blocks[s].block.rstrip().splitlines()) - lines.append("```") - lines.append("") - - lines.append(REF_END) - lines.append("") - return "\n".join(lines) - - -def update_curated_page(path: Path, family: str, specs: list[str], blocks: dict[str, ModuleBlock]) -> bool: - text = read_text(path) - - if REF_START in text and REF_END in text: - before, rest = text.split(REF_START, 1) - _, after = rest.split(REF_END, 1) - before = re.sub(r"(?:\n## API reference\s*)+\Z", "\n", before.rstrip()) - # Keep content outside the generated region; regenerate the region itself. - next_text = before.rstrip() + "\n\n" + render_reference_section(family, specs, blocks) + after.lstrip() - else: - next_text = text.rstrip() + "\n\n" + render_reference_section(family, specs, blocks) - - if "\n## See also\n" not in next_text and not next_text.startswith("## See also\n"): - next_text = next_text.rstrip() + "\n\n" + render_see_also_section() - - # Ensure trailing newline - next_text = next_text.rstrip() + "\n" - return write_text_if_changed(path, next_text) - - -def render_generated_page(path: Path, family: str, specs: list[str], blocks: dict[str, ModuleBlock]) -> str: - title = family_title(family, blocks) - specs_sorted = sort_specs_in_family(family, specs) - - lines: list[str] = [] - lines.append(f"# {title}") - lines.append("") - lines.append(family_intro(family).rstrip()) - lines.append("") - related = render_related_guides_section(family) - if related: - lines.append(related.rstrip()) - lines.append("") - lines.append(render_examples_section(family).rstrip()) - lines.append("") - lines.append(render_reference_section(family, specs_sorted, blocks).rstrip()) - lines.append("") - lines.append(render_see_also_section().rstrip()) - lines.append("") - return "\n".join(lines).rstrip() + "\n" - - -def main() -> None: - root = repo_root() - parser = argparse.ArgumentParser(description="Refresh Runtime API Markdown while preserving curated prose.") - parser.add_argument("--runtime-repo", type=Path, default=root.parent / "runtime") - args = parser.parse_args() - index_path = args.runtime_repo.resolve() / "api" / "index.d.ts" - if not index_path.exists(): - raise SystemExit(f"Missing {index_path}") - - blocks = parse_index_d_ts(read_text(index_path)) - - families: dict[str, list[str]] = {} - for spec in blocks.keys(): - families.setdefault(family_of(spec), []).append(spec) - - staging = tempfile.TemporaryDirectory(prefix="oro-runtime-import-") - staged = Path(staging.name) - source_pages.stage_collection(root, "runtime", staged) - out_dir = staged / "javascript" - out_dir.mkdir(parents=True, exist_ok=True) - - written: list[Path] = [] - removed: list[Path] = [] - - for family in sorted(families.keys()): - if family in EXCLUDED_PUBLIC_FAMILIES: - continue - - specs = families[family] - - if family in CURATED_FAMILIES: - rel = CURATED_FILES_BY_FAMILY[family] - path = out_dir / rel - if not path.exists(): - raise SystemExit(f"Expected curated file to exist: {path}") - if update_curated_page(path, family, specs, blocks): - written.append(path) - continue - - name = family.removeprefix("oro:") - path = out_dir / f"{name}.md" - text = render_generated_page(path, family, specs, blocks) - if write_text_if_changed(path, text): - written.append(path) - - for family in sorted(EXCLUDED_PUBLIC_FAMILIES): - stale_path = out_dir / f"{family.removeprefix('oro:')}.md" - if stale_path.exists(): - stale_path.unlink() - removed.append(stale_path) - - source_pages.import_collection(root, "runtime", staged) - staging.cleanup() - - if written or removed: - print("Wrote:") - for p in written: - print(f"- {root / 'src/runtime/docs/javascript' / p.stem / 'page.md'}") - for p in removed: - print(f"- removed {root / 'src/runtime/docs/javascript' / p.stem / 'page.md'}") - else: - print("Unchanged: generated JavaScript API reference pages") - - -if __name__ == "__main__": - main() diff --git a/runtime/tools/generate-js-api-reference.ts b/runtime/tools/generate-js-api-reference.ts new file mode 100644 index 00000000..6e13325d --- /dev/null +++ b/runtime/tools/generate-js-api-reference.ts @@ -0,0 +1,132 @@ +#!/usr/bin/env node +import { access, mkdir, mkdtemp, rm, unlink } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { resolve, join, basename } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { parseArgs } from 'node:util' +import { stageCollection, importCollection, readText, writeTextIfChanged, splitLines } from '../../tools/source-pages.ts' +import { DEFAULT_SEE_ALSO, DESCRIPTION_BY_FAMILY, EXAMPLES_BY_FAMILY, GUIDE_REFS_BY_FAMILY } from './js-api-reference-content.ts' + +export const CURATED_FILES_BY_FAMILY: Record = Object.fromEntries( + ['ai', 'application', 'extension', 'fs', 'hooks', 'mcp', 'notification', 'secure-storage', 'window'].map(name => [`oro:${name}`, `${name}.md`])) +export const EXCLUDED_PUBLIC_FAMILIES = new Set(['oro:bootstrap', 'oro:external', 'oro:internal', 'oro:node', 'oro:node-esm-loader']) +export const REF_START = '' +export const REF_END = '' +export interface ModuleBlock { spec: string; block: string } +export function parseIndexDTS(text: string): Record { + const lines = splitLines(text) + const blocks: Record = {} + for (let i = 0; i < lines.length; i++) { + const match = /^declare module ['"](oro:[^'"]+)['"]\s*\{\s*$/.exec(lines[i]) + if (!match) continue + const spec = match[1].trim() + const buf = [lines[i].trimEnd()] + while (++i < lines.length) { + const line = lines[i].trimEnd() + buf.push(line) + if (line === '}' && !/^[ \t]/.test(lines[i])) break + } + blocks[spec] = { spec, block: buf.join('\n').trimEnd() + '\n' } + } + if (!Object.keys(blocks).length) throw new Error("No `declare module 'oro:*' { ... }` blocks found.") + return blocks +} +export function familyOf(spec: string): string { return spec.split('/', 1)[0] } +export function sortSpecsInFamily(family: string, specs: string[]): string[] { + return [...specs].sort().sort((a, b) => Number(a !== family) - Number(b !== family)) +} +export function preferredImportSpec(family: string, specs: string[], blocks: Record): string { + return family in blocks ? family : specs.find(s => s.endsWith('/index')) ?? specs[0] +} +export function familyTitle(family: string, blocks: Record): string { + return '`' + family + (family in blocks ? '' : '/*') + '`' +} +export function familyIntro(family: string): string { + return DESCRIPTION_BY_FAMILY[family] ?? `This page documents the exported JavaScript surface for \`${family}\` as declared by the runtime’s published TypeScript definitions.\n` +} +export function renderExamplesSection(family: string): string { + const body = EXAMPLES_BY_FAMILY[family] + if (!body) throw new Error(`Missing generated examples for ${family}`) + return `## Examples\n\n${body.trimEnd()}\n` +} +export function renderSeeAlsoSection(): string { + return ['## See also', '', ...DEFAULT_SEE_ALSO.map(([label, id]) => `- [${label}](?p=${id})`), ''].join('\n') +} +export function renderRelatedGuidesSection(family: string): string { + const refs = GUIDE_REFS_BY_FAMILY[family] ?? [] + return refs.length ? ['## Related guides', '', ...refs.map(([label, id]) => `- [${label}](?p=${id})`), ''].join('\n') : '' +} +export function renderReferenceSection(family: string, specs: string[], blocks: Record): string { + const sorted = sortSpecsInFamily(family, specs) + const lines = ['## API reference', '', REF_START, '', '### Module specifiers', '', '```text', ...sorted, '```', '', + '### TypeScript declarations', '', "These declarations are generated from the runtime's published TypeScript surface.", ''] + for (const spec of sorted) lines.push(`#### \`${spec}\``, '', '```ts', ...splitLines(blocks[spec].block.trimEnd()), '```', '') + return [...lines, REF_END, ''].join('\n') +} +export async function updateCuratedPage(path: string, family: string, specs: string[], blocks: Record): Promise { + const text = await readText(path) + let next: string + if (text.includes(REF_START) && text.includes(REF_END)) { + const start = text.indexOf(REF_START) + const end = text.indexOf(REF_END, start + REF_START.length) + if (end < 0) throw new Error(`Reversed generated reference markers in ${path}`) + const before = text.slice(0, start).trimEnd().replace(/(?:\n## API reference\s*)+$/, '\n') + next = before.trimEnd() + '\n\n' + renderReferenceSection(family, specs, blocks) + text.slice(end + REF_END.length).trimStart() + } else next = text.trimEnd() + '\n\n' + renderReferenceSection(family, specs, blocks) + if (!next.includes('\n## See also\n') && !next.startsWith('## See also\n')) next = next.trimEnd() + '\n\n' + renderSeeAlsoSection() + return writeTextIfChanged(path, next.trimEnd() + '\n') +} +export function renderGeneratedPage(_path: string, family: string, specs: string[], blocks: Record): string { + const lines = [`# ${familyTitle(family, blocks)}`, '', familyIntro(family).trimEnd(), ''] + const related = renderRelatedGuidesSection(family) + if (related) lines.push(related.trimEnd(), '') + lines.push(renderExamplesSection(family).trimEnd(), '', renderReferenceSection(family, specs, blocks).trimEnd(), '', renderSeeAlsoSection().trimEnd(), '') + return lines.join('\n').trimEnd() + '\n' +} +export async function generateReference(site: string, runtimeRepo: string): Promise { + const index = resolve(runtimeRepo, 'api/index.d.ts') + try { await access(index) } catch { throw new Error(`Missing ${index}`) } + const blocks = parseIndexDTS(await readText(index)) + const families: Record = {} + for (const spec of Object.keys(blocks)) (families[familyOf(spec)] ??= []).push(spec) + const staged = await mkdtemp(join(tmpdir(), 'oro-runtime-import-')) + const written: string[] = [], removed: string[] = [] + try { + await stageCollection('runtime', staged, site) + const out = join(staged, 'javascript') + await mkdir(out, { recursive: true }) + for (const family of Object.keys(families).sort()) { + if (EXCLUDED_PUBLIC_FAMILIES.has(family)) continue + const curated = CURATED_FILES_BY_FAMILY[family] + const path = join(out, curated ?? `${family.slice(4)}.md`) + if (curated) { + try { await access(path) } catch { throw new Error(`Expected curated file to exist: ${path}`) } + if (await updateCuratedPage(path, family, families[family], blocks)) written.push(path) + } else if (await writeTextIfChanged(path, renderGeneratedPage(path, family, families[family], blocks))) written.push(path) + } + for (const family of [...EXCLUDED_PUBLIC_FAMILIES].sort()) { + const path = join(out, `${family.slice(4)}.md`) + try { await unlink(path); removed.push(path) } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + } + await importCollection('runtime', staged, site) + } finally { await rm(staged, { recursive: true, force: true }) } + if (written.length || removed.length) { + console.log('Wrote:') + for (const path of written) console.log(`- ${join(site, 'src/runtime/docs/javascript', basename(path, '.md'), 'page.md')}`) + for (const path of removed) console.log(`- removed ${join(site, 'src/runtime/docs/javascript', basename(path, '.md'), 'page.md')}`) + } else console.log('Unchanged: generated JavaScript API reference pages') +} +export async function main(args = process.argv.slice(2)): Promise { + const { values } = parseArgs({ args, options: { 'runtime-repo': { type: 'string' }, help: { type: 'boolean', short: 'h' } } }) + if (values.help) { console.log('Refresh Runtime API Markdown while preserving curated prose.\n\nUsage: node runtime/tools/generate-js-api-reference.ts [--runtime-repo PATH]'); return } + const site = fileURLToPath(new URL('../../', import.meta.url)) + await generateReference(site, values['runtime-repo'] ?? resolve(site, '../runtime')) +} +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { + console.error(error.message) + process.exitCode = error.code?.startsWith('ERR_PARSE_ARGS_') ? 2 : 1 + }) +} diff --git a/runtime/tools/js-api-reference-content.ts b/runtime/tools/js-api-reference-content.ts new file mode 100644 index 00000000..06280c58 --- /dev/null +++ b/runtime/tools/js-api-reference-content.ts @@ -0,0 +1,409 @@ +// Evaluated content from the original Python generator; preserve strings exactly. +export const DESCRIPTION_BY_FAMILY: Record = { + "oro:asn1": "`oro:asn1` parses ASN.1 source text and files into structured module data.", + "oro:assert": "`oro:assert` provides Node-compatible assertion helpers for tests, runtime checks, and invariants.", + "oro:async": "`oro:async` exposes async context propagation, deferred values, and resource tracking primitives.", + "oro:async_hooks": "`oro:async_hooks` exposes hook-based async lifecycle inspection compatible with Node-style instrumentation.", + "oro:background": "`oro:background` lets you inspect and coordinate the runtime background task surface.", + "oro:buffer": "`oro:buffer` provides `Buffer`, `Blob`, and related byte-oriented helpers.", + "oro:cdp": "`oro:cdp` controls the runtime Chrome DevTools Protocol endpoint used for inspection and debugging.", + "oro:child_process": "`oro:child_process` launches and manages subprocesses with a Node-compatible API.", + "oro:clipboard": "`oro:clipboard` reads and writes clipboard text from the current application context.", + "oro:commonjs": "`oro:commonjs/*` exposes the runtime’s CommonJS loader, cache, and resolution helpers.", + "oro:conduit": "`oro:conduit` manages the runtime WebSocket conduit used for internal cross-surface messaging.", + "oro:console": "`oro:console` exposes the runtime console implementation and patch helpers.", + "oro:constants": "`oro:constants` publishes the runtime’s platform, errno, and low-level constant tables.", + "oro:cookies": "`oro:cookies` reads and writes cookies for application-controlled origins.", + "oro:crypto": "`oro:crypto` exposes hashing, random bytes, and bundled sodium-backed helpers.", + "oro:dbus": "`oro:dbus` connects to DBus and lets JavaScript code call methods, watch signals, and manage names.", + "oro:dgram": "`oro:dgram` provides UDP sockets and multicast helpers with a Node-compatible API.", + "oro:diagnostics": "`oro:diagnostics` exposes channels, metrics, and runtime/window diagnostic queries.", + "oro:did": "`oro:did` parses and manipulates decentralized identifiers and DID URLs.", + "oro:dns": "`oro:dns` provides DNS lookup helpers and promise-based resolution APIs.", + "oro:enumeration": "`oro:enumeration` is a small typed-set helper used throughout the runtime surface.", + "oro:errno": "`oro:errno` converts errno values to names and messages.", + "oro:errors": "`oro:errors` exports DOM-style and runtime-specific error classes and constants.", + "oro:events": "`oro:events` provides `EventEmitter`, `EventTarget`, and event primitives used across the runtime.", + "oro:extension": "`oro:extension` loads native and wasm extensions and reports extension runtime state.", + "oro:fetch": "`oro:fetch` exposes the runtime fetch stack and standard request/response primitives.", + "oro:gc": "`oro:gc` lets you register finalizers and control object retention hooks.", + "oro:hci": "`oro:hci` exposes host controller interface helpers for low-level Bluetooth adapter access.", + "oro:http": "`oro:http` provides Node-compatible HTTP client and server APIs.", + "oro:https": "`oro:https` provides HTTPS client and server APIs on top of the runtime TLS stack.", + "oro:i18n": "`oro:i18n` resolves UI languages, localized messages, and locale-aware metadata.", + "oro:ip": "`oro:ip` normalizes and validates IP address inputs.", + "oro:ipc": "`oro:ipc` exposes low-level runtime IPC helpers used to talk to the native host process.", + "oro:ipfs": "`oro:ipfs` manages the embedded IPFS node and common content operations.", + "oro:iroh": "`oro:iroh` exposes the runtime’s Iroh transport bindings for peer-to-peer connections and streams.", + "oro:language": "`oro:language` resolves language names, tags, and related metadata.", + "oro:latica": "`oro:latica/*` exposes the runtime’s Latica networking primitives, packet helpers, and crypto utilities.", + "oro:location": "`oro:location` normalizes the current runtime location and origin semantics across platforms.", + "oro:mime": "`oro:mime` looks up MIME types, extensions, and content-type metadata.", + "oro:module": "`oro:module` exposes module-loader helpers and builtin-module inspection.", + "oro:navigation": "`oro:navigation` surfaces the Navigation API state that the runtime makes available to application code.", + "oro:net": "`oro:net` provides Node-compatible TCP clients and servers.", + "oro:network": "`oro:network` exposes a higher-level networking surface built on the Latica stack.", + "oro:npm": "`oro:npm/*` supports package-resolution and service-worker plumbing for NPM-backed modules.", + "oro:os": "`oro:os` exposes platform, CPU, temporary-directory, and system metadata helpers.", + "oro:path": "`oro:path` provides cross-platform path manipulation utilities.", + "oro:process": "`oro:process` exposes runtime process state, env access, signals, and scheduling helpers.", + "oro:protocol-handlers": "`oro:protocol-handlers` exposes runtime helpers for custom protocol and service-worker routing.", + "oro:querystring": "`oro:querystring` parses and serializes URL query strings.", + "oro:semver": "`oro:semver` parses, validates, and compares semantic versions.", + "oro:service-worker": "`oro:service-worker/*` exposes the service-worker environment, lifecycle objects, and helpers.", + "oro:shared-worker": "`oro:shared-worker/*` exposes the shared-worker environment and runtime helpers.", + "oro:signal": "`oro:signal` exposes process-signal constants, conversion helpers, and event listeners.", + "oro:sqlite": "`oro:sqlite` opens SQLite databases and exposes sync and async query helpers.", + "oro:stream": "`oro:stream` exposes Node-style streams, Web streams, and pipeline helpers.", + "oro:string_decoder": "`oro:string_decoder` decodes split byte streams into text without corrupting multibyte characters.", + "oro:tar": "`oro:tar` opens, creates, streams, and extracts tar archives.", + "oro:tcp": "`oro:tcp` exposes lower-level TCP client and server helpers alongside the higher-level `oro:net` API.", + "oro:test": "`oro:test` provides the runtime test harness used by application and module tests.", + "oro:timers": "`oro:timers` exposes timer primitives, promise-based sleeps, and scheduler helpers.", + "oro:tls": "`oro:tls` exposes TLS sockets, servers, and certificate-pin helpers.", + "oro:toml": "`oro:toml` parses and serializes TOML documents.", + "oro:tty": "`oro:tty` exposes terminal streams and TTY detection helpers.", + "oro:url": "`oro:url` exposes the WHATWG URL implementation used by the runtime.", + "oro:usb": "`oro:usb` installs and exposes the runtime WebUSB surface for window contexts.", + "oro:util": "`oro:util` exposes inspection, debug, and type-checking helpers.", + "oro:vm": "`oro:vm` runs JavaScript in isolated runtime contexts and manages context windows and workers.", + "oro:worker": "`oro:worker` re-exports the runtime worker classes for dedicated, shared, and service workers.", + "oro:worker_threads": "`oro:worker_threads` exposes worker-thread primitives, ports, and environment data helpers.", + "oro:xpc": "`oro:xpc` connects to macOS XPC services and encodes structured XPC payloads.", + "oro:zlib": "`oro:zlib` compresses and decompresses buffers and exposes zlib stream helpers." +} +export const EXAMPLES_BY_FAMILY: Record = { + "oro:asn1": "Parse ASN.1 source text and inspect the discovered modules:\n\n```js\nimport { parse } from 'oro:asn1'\n\nconst document = await parse(`\nDemo DEFINITIONS ::= BEGIN\n serialNumber INTEGER ::= 42\nEND\n`)\n\nconsole.log(document.modulesCount)\nconsole.log(document.modules[0]?.name)\n```\n", + "oro:assert": "Use assertion helpers for runtime invariants or tests:\n\n```js\nimport { ok, strictEqual, deepEqual } from 'oro:assert'\n\nconst payload = { id: 7, tags: ['runtime', 'docs'] }\n\nok(payload.id > 0)\nstrictEqual(payload.tags.length, 2)\ndeepEqual(payload.tags, ['runtime', 'docs'])\n```\n", + "oro:async": "Propagate request-scoped state across async boundaries:\n\n```js\nimport { AsyncLocalStorage } from 'oro:async'\n\nconst store = new AsyncLocalStorage()\n\nawait store.run({ requestId: 'req-42' }, async () => {\n await Promise.resolve()\n console.log(store.getStore().requestId)\n})\n```\n", + "oro:async_hooks": "Inspect async-resource lifecycles as work is scheduled:\n\n```js\nimport { createHook, executionAsyncId } from 'oro:async_hooks'\n\nconst hook = createHook({\n init(asyncId, type, triggerAsyncId) {\n console.log({ asyncId, type, triggerAsyncId })\n }\n})\n\nhook.enable()\nqueueMicrotask(() => {\n console.log(executionAsyncId())\n hook.disable()\n})\n```\n", + "oro:background": "Check whether background execution is available before scheduling work:\n\n```js\nimport background from 'oro:background'\n\nif (background.available) {\n console.log(await background.status())\n}\n```\n", + "oro:buffer": "Create byte buffers and wrap them in higher-level binary containers:\n\n```js\nimport Buffer, { Blob } from 'oro:buffer'\n\nconst bytes = Buffer.from('hello runtime', 'utf8')\nconst payload = new Blob([bytes], { type: 'text/plain' })\n\nconsole.log(bytes.length)\nconsole.log(payload.size)\n```\n", + "oro:cdp": "Inspect the current DevTools state and start the endpoint when needed:\n\n```js\nimport { status, listen, close } from 'oro:cdp'\n\nconsole.log(await status())\nawait listen({ host: '127.0.0.1', port: 9222 })\n\n// ...connect a DevTools client...\n\nawait close()\n```\n", + "oro:child_process": "Spawn a subprocess and consume its stdout stream:\n\n```js\nimport { spawn } from 'oro:child_process'\n\nconst child = spawn('echo', ['hello from runtime'])\n\nchild.stdout.on('data', (chunk) => {\n console.log(chunk.toString())\n})\n\nawait new Promise((resolve, reject) => {\n child.once('close', resolve)\n child.once('error', reject)\n})\n```\n", + "oro:clipboard": "Read and write plain-text clipboard content:\n\n```js\nimport { writeText, readText, canWriteText } from 'oro:clipboard'\n\nif (canWriteText()) {\n await writeText('Copied from Oro Runtime')\n console.log(await readText())\n}\n```\n", + "oro:commonjs": "Resolve CommonJS modules from the current ES module:\n\n```js\nimport { Module, isBuiltin } from 'oro:commonjs'\n\nconst require = Module.createRequire(import.meta.url)\n\nconsole.log(isBuiltin('fs'))\nconsole.log(require('./package.json').name)\n```\n", + "oro:conduit": "Connect to the runtime conduit and inspect its current status:\n\n```js\nimport { Conduit } from 'oro:conduit'\n\nconst conduit = new Conduit({ id: 'docs' })\nawait conduit.connect()\n\nconsole.log(await Conduit.status())\n\nawait conduit.close()\n```\n", + "oro:console": "Patch the global console so application logs use the runtime console implementation:\n\n```js\nimport { patchGlobalConsole } from 'oro:console'\n\npatchGlobalConsole(globalThis.console)\n\nconsole.info('runtime console is active')\n```\n", + "oro:constants": "Use named constants instead of hard-coding errno or signal values:\n\n```js\nimport { EACCES, SIGTERM } from 'oro:constants'\n\nconsole.log({ EACCES, SIGTERM })\n```\n", + "oro:cookies": "Set, read, and remove cookies for an application-controlled origin:\n\n```js\nimport { set, get, remove } from 'oro:cookies'\n\nawait set('https://app.example', 'session=abc123; Path=/; HttpOnly')\nconsole.log(await get('https://app.example'))\nawait remove('https://app.example', 'session')\n```\n", + "oro:crypto": "Generate random bytes and hash them with the runtime digest helper:\n\n```js\nimport { randomBytes, createDigest, murmur3 } from 'oro:crypto'\n\nconst nonce = randomBytes(16)\nconst digest = await createDigest('sha256', nonce)\n\nconsole.log(digest.toString('hex'))\nconsole.log(murmur3('hello-runtime'))\n```\n", + "oro:dbus": "Check DBus availability and open a session-bus connection:\n\n```js\nimport { availability, connect, BUS } from 'oro:dbus'\n\nconst state = await availability()\n\nif (state.available) {\n const bus = await connect({ bus: BUS.SESSION })\n console.log(bus.id)\n await bus.close()\n}\n```\n", + "oro:dgram": "Create a UDP socket and send a datagram:\n\n```js\nimport Buffer from 'oro:buffer'\nimport { createSocket } from 'oro:dgram'\n\nconst socket = createSocket('udp4')\n\nsocket.on('message', (message, rinfo) => {\n console.log(rinfo.address, message.toString())\n})\n\nsocket.bind(41234)\nsocket.send(Buffer.from('ping'), 41235, '127.0.0.1')\n```\n", + "oro:diagnostics": "Publish and subscribe to a diagnostics channel:\n\n```js\nimport { channel } from 'oro:diagnostics'\n\nconst buildChannel = channel('build')\n\nbuildChannel.subscribe('step', (message) => {\n console.log(message.step)\n})\n\nawait buildChannel.publish('step', { step: 'bundle-ready' })\n```\n", + "oro:did": "Parse a DID and a DID URL:\n\n```js\nimport { parse, parseUrl } from 'oro:did'\n\nconst did = parse('did:key:z6Mkkx...')\nconst url = parseUrl('did:key:z6Mkkx...#keys-1')\n\nconsole.log(did.method)\nconsole.log(url.fragment)\n```\n", + "oro:dns": "Resolve hostnames with the promise-based DNS helpers:\n\n```js\nimport { lookup } from 'oro:dns/promises'\n\nconst record = await lookup('oro.computer')\n\nconsole.log(record.address)\nconsole.log(record.family)\n```\n", + "oro:enumeration": "Use `Enumeration` when you want a typed, readable set of allowed values:\n\n```js\nimport Enumeration from 'oro:enumeration'\n\nconst states = new Enumeration(['draft', 'ready', 'shipped'])\n\nconsole.log(states.has('ready'))\nconsole.log([...states])\n```\n", + "oro:errno": "Convert errno values to names and human-readable messages:\n\n```js\nimport { EACCES, getName, getMessage } from 'oro:errno'\n\nconsole.log(getName(EACCES))\nconsole.log(getMessage(EACCES))\n```\n", + "oro:errors": "Construct runtime error types directly when surfacing failures:\n\n```js\nimport { AbortError, NotFoundError } from 'oro:errors'\n\nconst abort = new AbortError('request cancelled')\nconst missing = new NotFoundError('resource not found')\n\nconsole.log(abort.name, missing.name)\n```\n", + "oro:events": "Use `EventEmitter` for familiar event-based application code:\n\n```js\nimport EventEmitter from 'oro:events'\n\nconst bus = new EventEmitter()\n\nbus.on('ready', (value) => console.log(value))\nbus.emit('ready', { ok: true })\n```\n", + "oro:extension": "Load a runtime extension and inspect the resulting binding:\n\n```js\nimport { load, stats } from 'oro:extension'\n\nconst extension = await load('image-tools', { allow: ['resize'] })\n\nconsole.log(extension.type)\nconsole.log(await stats())\n\nawait extension.unload()\n```\n", + "oro:fetch": "Use the runtime fetch stack exactly like standard web fetch:\n\n```js\nimport fetch from 'oro:fetch'\n\nconst response = await fetch('https://example.com/api/status')\nconst data = await response.json()\n\nconsole.log(data)\n```\n", + "oro:gc": "Register a finalizer on an object that owns runtime resources:\n\n```js\nimport { ref, unref } from 'oro:gc'\n\nconst resource = {\n [Symbol.for('oro.runtime.gc.finalize')]() {\n console.log('cleaning up resource')\n }\n}\n\nref(resource)\nunref(resource)\n```\n", + "oro:hci": "Inspect local Bluetooth adapters before opening a low-level HCI socket:\n\n```js\nimport { listAdapters, getAdapter } from 'oro:hci'\n\nconst adapters = listAdapters()\nconsole.log(adapters)\n\nif (adapters.length > 0) {\n console.log(getAdapter(adapters[0].devId))\n}\n```\n", + "oro:http": "Start a small HTTP server with the Node-compatible API:\n\n```js\nimport { createServer } from 'oro:http'\n\nconst server = createServer((req, res) => {\n res.writeHead(200, { 'content-type': 'text/plain' })\n res.end('hello from oro:http')\n})\n\nserver.listen(8080, '127.0.0.1')\n```\n", + "oro:https": "Issue an HTTPS request with the familiar client helper:\n\n```js\nimport process from 'oro:process'\nimport { get } from 'oro:https'\n\nget('https://example.com', (res) => {\n res.on('data', (chunk) => process.stdout.write(chunk))\n})\n```\n", + "oro:i18n": "Resolve the current UI language and look up a localized message:\n\n```js\nimport { getUILanguage, getMessage } from 'oro:i18n'\n\nconsole.log(getUILanguage())\nconsole.log(getMessage('menu.file.open'))\n```\n", + "oro:ip": "Normalize user-provided IP input before using it in socket code:\n\n```js\nimport { normalizeIPv4, isIPv4 } from 'oro:ip'\n\nconst address = normalizeIPv4('127.000.000.001')\n\nconsole.log(address)\nconsole.log(isIPv4(address))\n```\n", + "oro:ipc": "Wait for the runtime IPC bridge and post a structured message:\n\n```js\nimport { ready, postMessage, debug } from 'oro:ipc'\n\ndebug(true)\nawait ready()\n\npostMessage({ type: 'docs-demo', at: Date.now() })\n```\n", + "oro:ipfs": "Ensure the embedded IPFS node is running before adding local content:\n\n```js\nimport { ensureStarted, add, status } from 'oro:ipfs'\n\nawait ensureStarted({ repoPath: './ipfs-repo' })\nconsole.log(await status())\n\nconst { cid } = await add('/absolute/path/to/report.txt')\nconsole.log(cid)\n```\n", + "oro:iroh": "Initialize the Iroh transport and create an endpoint:\n\n```js\nimport { ensureInitialized, Endpoint } from 'oro:iroh'\n\nawait ensureInitialized()\n\nconst endpoint = await Endpoint.create()\nawait endpoint.bind()\n\nconsole.log(await endpoint.nodeAddr())\n\nawait endpoint.close()\n```\n", + "oro:language": "Resolve language tags and metadata from user input:\n\n```js\nimport { lookup, describe } from 'oro:language'\n\nconsole.log(lookup('en-US'))\nconsole.log(describe('English'))\n```\n", + "oro:latica": "Use Latica helpers directly when you are working at the packet and transport layer:\n\n```js\nimport { sha256 } from 'oro:latica'\n\nconst topic = await sha256(new TextEncoder().encode('docs-demo'))\n\nconsole.log(topic)\n```\n", + "oro:location": "Inspect the runtime-normalized location values for the current context:\n\n```js\nimport location from 'oro:location'\n\nconsole.log(location.href)\nconsole.log(location.origin)\n```\n", + "oro:mime": "Look up MIME metadata for a type or extension:\n\n```js\nimport { lookupSync } from 'oro:mime'\n\nconsole.log(lookupSync('application/json'))\nconsole.log(lookupSync('.png'))\n```\n", + "oro:module": "Inspect builtin modules and create a `require()` function when you need one:\n\n```js\nimport Module, { builtinModules, isBuiltin, createRequire } from 'oro:module'\n\nconst require = createRequire(import.meta.url)\n\nconsole.log(isBuiltin('oro:path'))\nconsole.log(Object.keys(builtinModules).length)\nconsole.log(typeof Module)\nconsole.log(typeof require)\n```\n", + "oro:navigation": "Read the current navigation entry and listen for browser-style navigation events:\n\n```js\nimport navigation from 'oro:navigation'\n\nconsole.log(navigation.currentEntry?.url)\n\nnavigation.addEventListener('navigate', (event) => {\n console.log(event.destination.url)\n})\n```\n", + "oro:net": "Build a TCP echo server with the Node-compatible net API:\n\n```js\nimport { createServer } from 'oro:net'\n\nconst server = createServer({}, (socket) => {\n socket.on('data', (chunk) => socket.write(chunk))\n})\n\nserver.listen(4040, '127.0.0.1')\n```\n", + "oro:network": "Start the higher-level networking surface and attach event listeners:\n\n```js\nimport network from 'oro:network'\n\nconst bus = await network({})\n\nbus.on('error', console.error)\nbus.on('message', (message) => console.log(message))\n```\n", + "oro:npm": "Resolve an NPM specifier to the runtime module URL it will load:\n\n```js\nimport { resolve } from 'oro:npm/module'\n\nconst resolved = resolve('react', import.meta.url)\n\nconsole.log(resolved?.url)\nconsole.log(resolved?.type)\n```\n", + "oro:os": "Inspect the current platform and locate scratch space:\n\n```js\nimport { arch, platform, tmpdir, cpus } from 'oro:os'\n\nconsole.log(platform(), arch())\nconsole.log(tmpdir())\nconsole.log(cpus().length)\n```\n", + "oro:path": "Compose paths without branching on platform separators:\n\n```js\nimport { resolve, join, dirname, basename } from 'oro:path'\n\nconst absolute = resolve('dist', 'assets', 'logo.svg')\n\nconsole.log(absolute)\nconsole.log(dirname(absolute))\nconsole.log(basename(join('/tmp', 'report.txt')))\n```\n", + "oro:process": "Inspect process state and schedule work on the next tick:\n\n```js\nimport process, { env, hrtime, nextTick } from 'oro:process'\n\nconsole.log(env.PATH)\nconsole.log(hrtime())\n\nnextTick(() => {\n console.log(process.pid)\n})\n```\n", + "oro:protocol-handlers": "Ask the runtime which service worker is currently handling a custom scheme:\n\n```js\nimport { getServiceWorker } from 'oro:protocol-handlers'\n\nconst worker = await getServiceWorker({ scheme: 'npm' })\n\nconsole.log(worker)\n```\n", + "oro:querystring": "Parse and serialize query strings with the Node-compatible helpers:\n\n```js\nimport { stringify, parse } from 'oro:querystring'\n\nconst query = stringify({ q: 'oro runtime', page: 2 })\n\nconsole.log(query)\nconsole.log(parse(query))\n```\n", + "oro:semver": "Validate and compare versions before running upgrade logic:\n\n```js\nimport { valid, compare, gt } from 'oro:semver'\n\nconsole.log(valid('1.4.0'))\nconsole.log(compare('1.4.0', '1.3.9'))\nconsole.log(gt('2.0.0', '1.9.0'))\n```\n", + "oro:service-worker": "Use the runtime service-worker helpers from inside a service-worker scope:\n\n```js\nimport { env } from 'oro:service-worker'\n\nif (env) {\n self.addEventListener('fetch', (event) => {\n event.respondWith(fetch(event.request))\n })\n}\n```\n", + "oro:shared-worker": "Launch a shared worker and communicate through its shared port:\n\n```js\nimport { SharedWorker } from 'oro:shared-worker'\n\nconst worker = new SharedWorker(new URL('./shared.js', import.meta.url))\n\nworker.port.start()\nworker.port.postMessage({ type: 'warmup' })\n```\n", + "oro:signal": "Listen for runtime process signals using the signal helper surface:\n\n```js\nimport signal from 'oro:signal'\n\nsignal.addEventListener('SIGTERM', (event) => {\n console.log(event.type)\n})\n```\n", + "oro:sqlite": "Open an in-memory database and run a few synchronous queries:\n\n```js\nimport { open, OPEN_CREATE, OPEN_MEMORY } from 'oro:sqlite'\n\nconst db = open(':memory:', { flags: OPEN_CREATE | OPEN_MEMORY })\n\ndb.exec('create table notes (id integer primary key, title text)')\ndb.exec(\"insert into notes (title) values ('hello')\")\n\nconsole.log(db.query('select * from notes'))\n```\n", + "oro:stream": "Connect streams together with the pipeline helper:\n\n```js\nimport process from 'oro:process'\nimport { Readable, Writable, pipelinePromise } from 'oro:stream'\n\nconst source = Readable.from(['oro', ' ', 'runtime'])\nconst sink = new Writable({\n write(chunk, _encoding, done) {\n process.stdout.write(chunk)\n done()\n }\n})\n\nawait pipelinePromise(source, sink)\n```\n", + "oro:string_decoder": "Decode chunked UTF-8 data without splitting multibyte characters:\n\n```js\nimport Buffer from 'oro:buffer'\nimport StringDecoder from 'oro:string_decoder'\n\nconst decoder = new StringDecoder('utf8')\nconst chunks = [Buffer.from([0xe2, 0x82]), Buffer.from([0xac])]\n\nconsole.log(decoder.write(chunks[0]))\nconsole.log(decoder.write(chunks[1]))\n```\n", + "oro:tar": "Build a tar archive in memory and inspect its entries:\n\n```js\nimport Buffer from 'oro:buffer'\nimport { createInMemory } from 'oro:tar'\n\nconst archive = await createInMemory()\n\nawait archive.append({ path: 'hello.txt' }, Buffer.from('hello'))\nawait archive.finalize()\n\nconsole.log(await archive.entries())\n```\n", + "oro:tcp": "Open a lower-level TCP server when you want more direct socket control:\n\n```js\nimport { createServer } from 'oro:tcp'\n\nconst server = createServer((socket) => {\n socket.write(Buffer.from('connected'))\n})\n\nserver.listen({ port: 5050, host: '127.0.0.1' })\n```\n", + "oro:test": "Write runtime tests with the built-in test harness:\n\n```js\nimport test from 'oro:test'\n\ntest('querystring roundtrip', async (t) => {\n t.equal('a=1', new URLSearchParams({ a: '1' }).toString())\n})\n```\n", + "oro:timers": "Sleep without writing your own timeout wrapper:\n\n```js\nimport { sleep } from 'oro:timers'\n\nawait sleep(250)\n\nconsole.log('quarter second elapsed')\n```\n", + "oro:tls": "Configure TLS pins before opening pinned connections:\n\n```js\nimport { setTlsPins, getTlsPins, clearTlsPins } from 'oro:tls'\n\nawait setTlsPins('sha256/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=')\nconsole.log(await getTlsPins())\nawait clearTlsPins()\n```\n", + "oro:toml": "Parse configuration and emit updated TOML back to disk:\n\n```js\nimport { parse, stringify } from 'oro:toml'\n\nconst config = parse('name = \"oro\"\\nport = 8080')\nconfig.debug = true\n\nconsole.log(stringify(config))\n```\n", + "oro:tty": "Detect interactive terminals before enabling richer output:\n\n```js\nimport { isatty } from 'oro:tty'\n\nconsole.log(isatty(0))\nconsole.log(isatty(1))\n```\n", + "oro:url": "Use the WHATWG URL implementation for robust URL parsing:\n\n```js\nimport URL from 'oro:url'\n\nconst url = new URL('https://oro.computer/docs?project=runtime')\n\nconsole.log(url.pathname)\nconsole.log(url.searchParams.get('project'))\n```\n", + "oro:usb": "Install the runtime WebUSB surface and request a device in window contexts:\n\n```js\nimport { installNavigatorUSB } from 'oro:usb'\n\nconst usb = installNavigatorUSB()\n\nif (usb) {\n console.log(await usb.getDevices())\n // const device = await usb.requestDevice({ filters: [{ vendorId: 0x1209 }] })\n}\n```\n", + "oro:util": "Use the runtime debug helper and common type predicates:\n\n```js\nimport { debug, isTypedArray } from 'oro:util'\n\nconst log = debug('docs')\nconst payload = new Uint8Array([1, 2, 3])\n\nlog('payload ready')\nconsole.log(isTypedArray(payload))\n```\n", + "oro:vm": "Run a script inside an isolated context object:\n\n```js\nimport { runInContext } from 'oro:vm'\n\nconst context = { answer: 41 }\nconst result = await runInContext('answer + 1', context)\n\nconsole.log(result)\n```\n", + "oro:worker": "Launch a dedicated worker through the runtime worker surface:\n\n```js\nimport Worker from 'oro:worker'\n\nconst worker = new Worker(new URL('./worker.js', import.meta.url), {\n workerData: { job: 'thumbnail' }\n})\n\nworker.postMessage({ type: 'start' })\n```\n", + "oro:worker_threads": "Pass environment data into worker threads and communicate with message channels:\n\n```js\nimport { MessageChannel, setEnvironmentData } from 'oro:worker_threads'\n\nsetEnvironmentData('traceId', 'req-42')\n\nconst { port1, port2 } = new MessageChannel()\nport1.onmessage = (event) => console.log(event.data)\nport2.postMessage({ ok: true })\n```\n", + "oro:xpc": "Open an XPC connection, send a request, and close it cleanly:\n\n```js\nimport { availability, connect } from 'oro:xpc'\n\nconst state = await availability()\n\nif (state.available) {\n const connection = await connect({ service: 'com.example.agent' })\n await connection.sendAndForget({ type: 'ping' })\n await connection.close()\n}\n```\n", + "oro:zlib": "Compress and decompress buffers with the promise-based helpers:\n\n```js\nimport { gzip, gunzip } from 'oro:zlib'\n\nconst compressed = await gzip('hello runtime')\nconst restored = await gunzip(compressed)\n\nconsole.log(restored.toString())\n```\n" +} +export const GUIDE_REFS_BY_FAMILY: Record = { + "oro:application": [ + [ + "Windows and messaging", + "guides/windows-and-messaging" + ], + [ + "Lifecycle hooks and deep links", + "guides/lifecycle-hooks-and-deep-links" + ], + [ + "Desktop integrations", + "guides/desktop-integrations" + ] + ], + "oro:assert": [ + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:cdp": [ + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:child_process": [ + [ + "Configure your runtime project", + "guides/configure-your-runtime-project" + ], + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:clipboard": [ + [ + "Desktop integrations", + "guides/desktop-integrations" + ] + ], + "oro:console": [ + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:crypto": [ + [ + "Secure storage and sessions", + "guides/secure-storage-and-sessions" + ] + ], + "oro:diagnostics": [ + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:errors": [ + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:extension": [ + [ + "Native extensions", + "guides/native-extensions" + ] + ], + "oro:fetch": [ + [ + "Calling HTTP APIs", + "guides/calling-http-apis" + ], + [ + "Offline-first with service workers", + "guides/offline-first-with-service-workers" + ] + ], + "oro:fs": [ + [ + "Files and sandboxing", + "guides/files-and-sandboxing" + ], + [ + "Project layout", + "guides/project-layout" + ] + ], + "oro:hooks": [ + [ + "Lifecycle hooks and deep links", + "guides/lifecycle-hooks-and-deep-links" + ], + [ + "Windows and messaging", + "guides/windows-and-messaging" + ] + ], + "oro:http": [ + [ + "Calling HTTP APIs", + "guides/calling-http-apis" + ] + ], + "oro:https": [ + [ + "Calling HTTP APIs", + "guides/calling-http-apis" + ] + ], + "oro:ipc": [ + [ + "Native extensions", + "guides/native-extensions" + ], + [ + "Windows and messaging", + "guides/windows-and-messaging" + ] + ], + "oro:location": [ + [ + "Lifecycle hooks and deep links", + "guides/lifecycle-hooks-and-deep-links" + ] + ], + "oro:mcp": [ + [ + "MCP and agent automation", + "guides/mcp-and-agent-automation" + ] + ], + "oro:navigation": [ + [ + "Lifecycle hooks and deep links", + "guides/lifecycle-hooks-and-deep-links" + ], + [ + "Windows and messaging", + "guides/windows-and-messaging" + ] + ], + "oro:notification": [ + [ + "Desktop integrations", + "guides/desktop-integrations" + ], + [ + "Lifecycle hooks and deep links", + "guides/lifecycle-hooks-and-deep-links" + ] + ], + "oro:path": [ + [ + "Files and sandboxing", + "guides/files-and-sandboxing" + ], + [ + "Project layout", + "guides/project-layout" + ] + ], + "oro:process": [ + [ + "Configure your runtime project", + "guides/configure-your-runtime-project" + ] + ], + "oro:protocol-handlers": [ + [ + "Custom protocols and routing", + "guides/custom-protocols-and-routing" + ], + [ + "Offline-first with service workers", + "guides/offline-first-with-service-workers" + ] + ], + "oro:secure-storage": [ + [ + "Secure storage and sessions", + "guides/secure-storage-and-sessions" + ] + ], + "oro:service-worker": [ + [ + "Offline-first with service workers", + "guides/offline-first-with-service-workers" + ], + [ + "Custom protocols and routing", + "guides/custom-protocols-and-routing" + ] + ], + "oro:shared-worker": [ + [ + "Shared workers across windows", + "guides/shared-workers-across-windows" + ], + [ + "Windows and messaging", + "guides/windows-and-messaging" + ] + ], + "oro:sqlite": [ + [ + "Local data with SQLite", + "guides/local-data-with-sqlite" + ], + [ + "Files and sandboxing", + "guides/files-and-sandboxing" + ] + ], + "oro:test": [ + [ + "Testing and diagnostics", + "guides/testing-and-diagnostics" + ] + ], + "oro:worker": [ + [ + "Worker threads for heavy work", + "guides/worker-threads-for-heavy-work" + ], + [ + "Shared workers across windows", + "guides/shared-workers-across-windows" + ] + ], + "oro:worker_threads": [ + [ + "Worker threads for heavy work", + "guides/worker-threads-for-heavy-work" + ] + ], + "oro:window": [ + [ + "Windows and messaging", + "guides/windows-and-messaging" + ], + [ + "Desktop integrations", + "guides/desktop-integrations" + ] + ] +} +export const DEFAULT_SEE_ALSO: readonly (readonly [string, string])[] = [ + [ + "JavaScript APIs overview", + "javascript/overview" + ], + [ + "All module specifiers", + "javascript/all-modules" + ] +] diff --git a/runtime/tools/js_api_reference_content.py b/runtime/tools/js_api_reference_content.py deleted file mode 100644 index cca22c85..00000000 --- a/runtime/tools/js_api_reference_content.py +++ /dev/null @@ -1,1256 +0,0 @@ -from __future__ import annotations - -from textwrap import dedent - - -def md(text: str) -> str: - return dedent(text).strip() + "\n" - - -DESCRIPTION_BY_FAMILY: dict[str, str] = { - "oro:asn1": "`oro:asn1` parses ASN.1 source text and files into structured module data.", - "oro:assert": "`oro:assert` provides Node-compatible assertion helpers for tests, runtime checks, and invariants.", - "oro:async": "`oro:async` exposes async context propagation, deferred values, and resource tracking primitives.", - "oro:async_hooks": "`oro:async_hooks` exposes hook-based async lifecycle inspection compatible with Node-style instrumentation.", - "oro:background": "`oro:background` lets you inspect and coordinate the runtime background task surface.", - "oro:buffer": "`oro:buffer` provides `Buffer`, `Blob`, and related byte-oriented helpers.", - "oro:cdp": "`oro:cdp` controls the runtime Chrome DevTools Protocol endpoint used for inspection and debugging.", - "oro:child_process": "`oro:child_process` launches and manages subprocesses with a Node-compatible API.", - "oro:clipboard": "`oro:clipboard` reads and writes clipboard text from the current application context.", - "oro:commonjs": "`oro:commonjs/*` exposes the runtime’s CommonJS loader, cache, and resolution helpers.", - "oro:conduit": "`oro:conduit` manages the runtime WebSocket conduit used for internal cross-surface messaging.", - "oro:console": "`oro:console` exposes the runtime console implementation and patch helpers.", - "oro:constants": "`oro:constants` publishes the runtime’s platform, errno, and low-level constant tables.", - "oro:cookies": "`oro:cookies` reads and writes cookies for application-controlled origins.", - "oro:crypto": "`oro:crypto` exposes hashing, random bytes, and bundled sodium-backed helpers.", - "oro:dbus": "`oro:dbus` connects to DBus and lets JavaScript code call methods, watch signals, and manage names.", - "oro:dgram": "`oro:dgram` provides UDP sockets and multicast helpers with a Node-compatible API.", - "oro:diagnostics": "`oro:diagnostics` exposes channels, metrics, and runtime/window diagnostic queries.", - "oro:did": "`oro:did` parses and manipulates decentralized identifiers and DID URLs.", - "oro:dns": "`oro:dns` provides DNS lookup helpers and promise-based resolution APIs.", - "oro:enumeration": "`oro:enumeration` is a small typed-set helper used throughout the runtime surface.", - "oro:errno": "`oro:errno` converts errno values to names and messages.", - "oro:errors": "`oro:errors` exports DOM-style and runtime-specific error classes and constants.", - "oro:events": "`oro:events` provides `EventEmitter`, `EventTarget`, and event primitives used across the runtime.", - "oro:extension": "`oro:extension` loads native and wasm extensions and reports extension runtime state.", - "oro:fetch": "`oro:fetch` exposes the runtime fetch stack and standard request/response primitives.", - "oro:gc": "`oro:gc` lets you register finalizers and control object retention hooks.", - "oro:hci": "`oro:hci` exposes host controller interface helpers for low-level Bluetooth adapter access.", - "oro:http": "`oro:http` provides Node-compatible HTTP client and server APIs.", - "oro:https": "`oro:https` provides HTTPS client and server APIs on top of the runtime TLS stack.", - "oro:i18n": "`oro:i18n` resolves UI languages, localized messages, and locale-aware metadata.", - "oro:ip": "`oro:ip` normalizes and validates IP address inputs.", - "oro:ipc": "`oro:ipc` exposes low-level runtime IPC helpers used to talk to the native host process.", - "oro:ipfs": "`oro:ipfs` manages the embedded IPFS node and common content operations.", - "oro:iroh": "`oro:iroh` exposes the runtime’s Iroh transport bindings for peer-to-peer connections and streams.", - "oro:language": "`oro:language` resolves language names, tags, and related metadata.", - "oro:latica": "`oro:latica/*` exposes the runtime’s Latica networking primitives, packet helpers, and crypto utilities.", - "oro:location": "`oro:location` normalizes the current runtime location and origin semantics across platforms.", - "oro:mime": "`oro:mime` looks up MIME types, extensions, and content-type metadata.", - "oro:module": "`oro:module` exposes module-loader helpers and builtin-module inspection.", - "oro:navigation": "`oro:navigation` surfaces the Navigation API state that the runtime makes available to application code.", - "oro:net": "`oro:net` provides Node-compatible TCP clients and servers.", - "oro:network": "`oro:network` exposes a higher-level networking surface built on the Latica stack.", - "oro:npm": "`oro:npm/*` supports package-resolution and service-worker plumbing for NPM-backed modules.", - "oro:os": "`oro:os` exposes platform, CPU, temporary-directory, and system metadata helpers.", - "oro:path": "`oro:path` provides cross-platform path manipulation utilities.", - "oro:process": "`oro:process` exposes runtime process state, env access, signals, and scheduling helpers.", - "oro:protocol-handlers": "`oro:protocol-handlers` exposes runtime helpers for custom protocol and service-worker routing.", - "oro:querystring": "`oro:querystring` parses and serializes URL query strings.", - "oro:semver": "`oro:semver` parses, validates, and compares semantic versions.", - "oro:service-worker": "`oro:service-worker/*` exposes the service-worker environment, lifecycle objects, and helpers.", - "oro:shared-worker": "`oro:shared-worker/*` exposes the shared-worker environment and runtime helpers.", - "oro:signal": "`oro:signal` exposes process-signal constants, conversion helpers, and event listeners.", - "oro:sqlite": "`oro:sqlite` opens SQLite databases and exposes sync and async query helpers.", - "oro:stream": "`oro:stream` exposes Node-style streams, Web streams, and pipeline helpers.", - "oro:string_decoder": "`oro:string_decoder` decodes split byte streams into text without corrupting multibyte characters.", - "oro:tar": "`oro:tar` opens, creates, streams, and extracts tar archives.", - "oro:tcp": "`oro:tcp` exposes lower-level TCP client and server helpers alongside the higher-level `oro:net` API.", - "oro:test": "`oro:test` provides the runtime test harness used by application and module tests.", - "oro:timers": "`oro:timers` exposes timer primitives, promise-based sleeps, and scheduler helpers.", - "oro:tls": "`oro:tls` exposes TLS sockets, servers, and certificate-pin helpers.", - "oro:toml": "`oro:toml` parses and serializes TOML documents.", - "oro:tty": "`oro:tty` exposes terminal streams and TTY detection helpers.", - "oro:url": "`oro:url` exposes the WHATWG URL implementation used by the runtime.", - "oro:usb": "`oro:usb` installs and exposes the runtime WebUSB surface for window contexts.", - "oro:util": "`oro:util` exposes inspection, debug, and type-checking helpers.", - "oro:vm": "`oro:vm` runs JavaScript in isolated runtime contexts and manages context windows and workers.", - "oro:worker": "`oro:worker` re-exports the runtime worker classes for dedicated, shared, and service workers.", - "oro:worker_threads": "`oro:worker_threads` exposes worker-thread primitives, ports, and environment data helpers.", - "oro:xpc": "`oro:xpc` connects to macOS XPC services and encodes structured XPC payloads.", - "oro:zlib": "`oro:zlib` compresses and decompresses buffers and exposes zlib stream helpers.", -} - - -EXAMPLES_BY_FAMILY: dict[str, str] = { - "oro:asn1": md( - """ - Parse ASN.1 source text and inspect the discovered modules: - - ```js - import { parse } from 'oro:asn1' - - const document = await parse(` - Demo DEFINITIONS ::= BEGIN - serialNumber INTEGER ::= 42 - END - `) - - console.log(document.modulesCount) - console.log(document.modules[0]?.name) - ``` - """ - ), - "oro:assert": md( - """ - Use assertion helpers for runtime invariants or tests: - - ```js - import { ok, strictEqual, deepEqual } from 'oro:assert' - - const payload = { id: 7, tags: ['runtime', 'docs'] } - - ok(payload.id > 0) - strictEqual(payload.tags.length, 2) - deepEqual(payload.tags, ['runtime', 'docs']) - ``` - """ - ), - "oro:async": md( - """ - Propagate request-scoped state across async boundaries: - - ```js - import { AsyncLocalStorage } from 'oro:async' - - const store = new AsyncLocalStorage() - - await store.run({ requestId: 'req-42' }, async () => { - await Promise.resolve() - console.log(store.getStore().requestId) - }) - ``` - """ - ), - "oro:async_hooks": md( - """ - Inspect async-resource lifecycles as work is scheduled: - - ```js - import { createHook, executionAsyncId } from 'oro:async_hooks' - - const hook = createHook({ - init(asyncId, type, triggerAsyncId) { - console.log({ asyncId, type, triggerAsyncId }) - } - }) - - hook.enable() - queueMicrotask(() => { - console.log(executionAsyncId()) - hook.disable() - }) - ``` - """ - ), - "oro:background": md( - """ - Check whether background execution is available before scheduling work: - - ```js - import background from 'oro:background' - - if (background.available) { - console.log(await background.status()) - } - ``` - """ - ), - "oro:buffer": md( - """ - Create byte buffers and wrap them in higher-level binary containers: - - ```js - import Buffer, { Blob } from 'oro:buffer' - - const bytes = Buffer.from('hello runtime', 'utf8') - const payload = new Blob([bytes], { type: 'text/plain' }) - - console.log(bytes.length) - console.log(payload.size) - ``` - """ - ), - "oro:cdp": md( - """ - Inspect the current DevTools state and start the endpoint when needed: - - ```js - import { status, listen, close } from 'oro:cdp' - - console.log(await status()) - await listen({ host: '127.0.0.1', port: 9222 }) - - // ...connect a DevTools client... - - await close() - ``` - """ - ), - "oro:child_process": md( - """ - Spawn a subprocess and consume its stdout stream: - - ```js - import { spawn } from 'oro:child_process' - - const child = spawn('echo', ['hello from runtime']) - - child.stdout.on('data', (chunk) => { - console.log(chunk.toString()) - }) - - await new Promise((resolve, reject) => { - child.once('close', resolve) - child.once('error', reject) - }) - ``` - """ - ), - "oro:clipboard": md( - """ - Read and write plain-text clipboard content: - - ```js - import { writeText, readText, canWriteText } from 'oro:clipboard' - - if (canWriteText()) { - await writeText('Copied from Oro Runtime') - console.log(await readText()) - } - ``` - """ - ), - "oro:commonjs": md( - """ - Resolve CommonJS modules from the current ES module: - - ```js - import { Module, isBuiltin } from 'oro:commonjs' - - const require = Module.createRequire(import.meta.url) - - console.log(isBuiltin('fs')) - console.log(require('./package.json').name) - ``` - """ - ), - "oro:conduit": md( - """ - Connect to the runtime conduit and inspect its current status: - - ```js - import { Conduit } from 'oro:conduit' - - const conduit = new Conduit({ id: 'docs' }) - await conduit.connect() - - console.log(await Conduit.status()) - - await conduit.close() - ``` - """ - ), - "oro:console": md( - """ - Patch the global console so application logs use the runtime console implementation: - - ```js - import { patchGlobalConsole } from 'oro:console' - - patchGlobalConsole(globalThis.console) - - console.info('runtime console is active') - ``` - """ - ), - "oro:constants": md( - """ - Use named constants instead of hard-coding errno or signal values: - - ```js - import { EACCES, SIGTERM } from 'oro:constants' - - console.log({ EACCES, SIGTERM }) - ``` - """ - ), - "oro:cookies": md( - """ - Set, read, and remove cookies for an application-controlled origin: - - ```js - import { set, get, remove } from 'oro:cookies' - - await set('https://app.example', 'session=abc123; Path=/; HttpOnly') - console.log(await get('https://app.example')) - await remove('https://app.example', 'session') - ``` - """ - ), - "oro:crypto": md( - """ - Generate random bytes and hash them with the runtime digest helper: - - ```js - import { randomBytes, createDigest, murmur3 } from 'oro:crypto' - - const nonce = randomBytes(16) - const digest = await createDigest('sha256', nonce) - - console.log(digest.toString('hex')) - console.log(murmur3('hello-runtime')) - ``` - """ - ), - "oro:dbus": md( - """ - Check DBus availability and open a session-bus connection: - - ```js - import { availability, connect, BUS } from 'oro:dbus' - - const state = await availability() - - if (state.available) { - const bus = await connect({ bus: BUS.SESSION }) - console.log(bus.id) - await bus.close() - } - ``` - """ - ), - "oro:dgram": md( - """ - Create a UDP socket and send a datagram: - - ```js - import Buffer from 'oro:buffer' - import { createSocket } from 'oro:dgram' - - const socket = createSocket('udp4') - - socket.on('message', (message, rinfo) => { - console.log(rinfo.address, message.toString()) - }) - - socket.bind(41234) - socket.send(Buffer.from('ping'), 41235, '127.0.0.1') - ``` - """ - ), - "oro:diagnostics": md( - """ - Publish and subscribe to a diagnostics channel: - - ```js - import { channel } from 'oro:diagnostics' - - const buildChannel = channel('build') - - buildChannel.subscribe('step', (message) => { - console.log(message.step) - }) - - await buildChannel.publish('step', { step: 'bundle-ready' }) - ``` - """ - ), - "oro:did": md( - """ - Parse a DID and a DID URL: - - ```js - import { parse, parseUrl } from 'oro:did' - - const did = parse('did:key:z6Mkkx...') - const url = parseUrl('did:key:z6Mkkx...#keys-1') - - console.log(did.method) - console.log(url.fragment) - ``` - """ - ), - "oro:dns": md( - """ - Resolve hostnames with the promise-based DNS helpers: - - ```js - import { lookup } from 'oro:dns/promises' - - const record = await lookup('oro.computer') - - console.log(record.address) - console.log(record.family) - ``` - """ - ), - "oro:enumeration": md( - """ - Use `Enumeration` when you want a typed, readable set of allowed values: - - ```js - import Enumeration from 'oro:enumeration' - - const states = new Enumeration(['draft', 'ready', 'shipped']) - - console.log(states.has('ready')) - console.log([...states]) - ``` - """ - ), - "oro:errno": md( - """ - Convert errno values to names and human-readable messages: - - ```js - import { EACCES, getName, getMessage } from 'oro:errno' - - console.log(getName(EACCES)) - console.log(getMessage(EACCES)) - ``` - """ - ), - "oro:errors": md( - """ - Construct runtime error types directly when surfacing failures: - - ```js - import { AbortError, NotFoundError } from 'oro:errors' - - const abort = new AbortError('request cancelled') - const missing = new NotFoundError('resource not found') - - console.log(abort.name, missing.name) - ``` - """ - ), - "oro:events": md( - """ - Use `EventEmitter` for familiar event-based application code: - - ```js - import EventEmitter from 'oro:events' - - const bus = new EventEmitter() - - bus.on('ready', (value) => console.log(value)) - bus.emit('ready', { ok: true }) - ``` - """ - ), - "oro:extension": md( - """ - Load a runtime extension and inspect the resulting binding: - - ```js - import { load, stats } from 'oro:extension' - - const extension = await load('image-tools', { allow: ['resize'] }) - - console.log(extension.type) - console.log(await stats()) - - await extension.unload() - ``` - """ - ), - "oro:fetch": md( - """ - Use the runtime fetch stack exactly like standard web fetch: - - ```js - import fetch from 'oro:fetch' - - const response = await fetch('https://example.com/api/status') - const data = await response.json() - - console.log(data) - ``` - """ - ), - "oro:gc": md( - """ - Register a finalizer on an object that owns runtime resources: - - ```js - import { ref, unref } from 'oro:gc' - - const resource = { - [Symbol.for('oro.runtime.gc.finalize')]() { - console.log('cleaning up resource') - } - } - - ref(resource) - unref(resource) - ``` - """ - ), - "oro:hci": md( - """ - Inspect local Bluetooth adapters before opening a low-level HCI socket: - - ```js - import { listAdapters, getAdapter } from 'oro:hci' - - const adapters = listAdapters() - console.log(adapters) - - if (adapters.length > 0) { - console.log(getAdapter(adapters[0].devId)) - } - ``` - """ - ), - "oro:http": md( - """ - Start a small HTTP server with the Node-compatible API: - - ```js - import { createServer } from 'oro:http' - - const server = createServer((req, res) => { - res.writeHead(200, { 'content-type': 'text/plain' }) - res.end('hello from oro:http') - }) - - server.listen(8080, '127.0.0.1') - ``` - """ - ), - "oro:https": md( - """ - Issue an HTTPS request with the familiar client helper: - - ```js - import process from 'oro:process' - import { get } from 'oro:https' - - get('https://example.com', (res) => { - res.on('data', (chunk) => process.stdout.write(chunk)) - }) - ``` - """ - ), - "oro:i18n": md( - """ - Resolve the current UI language and look up a localized message: - - ```js - import { getUILanguage, getMessage } from 'oro:i18n' - - console.log(getUILanguage()) - console.log(getMessage('menu.file.open')) - ``` - """ - ), - "oro:ip": md( - """ - Normalize user-provided IP input before using it in socket code: - - ```js - import { normalizeIPv4, isIPv4 } from 'oro:ip' - - const address = normalizeIPv4('127.000.000.001') - - console.log(address) - console.log(isIPv4(address)) - ``` - """ - ), - "oro:ipc": md( - """ - Wait for the runtime IPC bridge and post a structured message: - - ```js - import { ready, postMessage, debug } from 'oro:ipc' - - debug(true) - await ready() - - postMessage({ type: 'docs-demo', at: Date.now() }) - ``` - """ - ), - "oro:ipfs": md( - """ - Ensure the embedded IPFS node is running before adding local content: - - ```js - import { ensureStarted, add, status } from 'oro:ipfs' - - await ensureStarted({ repoPath: './ipfs-repo' }) - console.log(await status()) - - const { cid } = await add('/absolute/path/to/report.txt') - console.log(cid) - ``` - """ - ), - "oro:iroh": md( - """ - Initialize the Iroh transport and create an endpoint: - - ```js - import { ensureInitialized, Endpoint } from 'oro:iroh' - - await ensureInitialized() - - const endpoint = await Endpoint.create() - await endpoint.bind() - - console.log(await endpoint.nodeAddr()) - - await endpoint.close() - ``` - """ - ), - "oro:language": md( - """ - Resolve language tags and metadata from user input: - - ```js - import { lookup, describe } from 'oro:language' - - console.log(lookup('en-US')) - console.log(describe('English')) - ``` - """ - ), - "oro:latica": md( - """ - Use Latica helpers directly when you are working at the packet and transport layer: - - ```js - import { sha256 } from 'oro:latica' - - const topic = await sha256(new TextEncoder().encode('docs-demo')) - - console.log(topic) - ``` - """ - ), - "oro:location": md( - """ - Inspect the runtime-normalized location values for the current context: - - ```js - import location from 'oro:location' - - console.log(location.href) - console.log(location.origin) - ``` - """ - ), - "oro:mime": md( - """ - Look up MIME metadata for a type or extension: - - ```js - import { lookupSync } from 'oro:mime' - - console.log(lookupSync('application/json')) - console.log(lookupSync('.png')) - ``` - """ - ), - "oro:module": md( - """ - Inspect builtin modules and create a `require()` function when you need one: - - ```js - import Module, { builtinModules, isBuiltin, createRequire } from 'oro:module' - - const require = createRequire(import.meta.url) - - console.log(isBuiltin('oro:path')) - console.log(Object.keys(builtinModules).length) - console.log(typeof Module) - console.log(typeof require) - ``` - """ - ), - "oro:navigation": md( - """ - Read the current navigation entry and listen for browser-style navigation events: - - ```js - import navigation from 'oro:navigation' - - console.log(navigation.currentEntry?.url) - - navigation.addEventListener('navigate', (event) => { - console.log(event.destination.url) - }) - ``` - """ - ), - "oro:net": md( - """ - Build a TCP echo server with the Node-compatible net API: - - ```js - import { createServer } from 'oro:net' - - const server = createServer({}, (socket) => { - socket.on('data', (chunk) => socket.write(chunk)) - }) - - server.listen(4040, '127.0.0.1') - ``` - """ - ), - "oro:network": md( - """ - Start the higher-level networking surface and attach event listeners: - - ```js - import network from 'oro:network' - - const bus = await network({}) - - bus.on('error', console.error) - bus.on('message', (message) => console.log(message)) - ``` - """ - ), - "oro:npm": md( - """ - Resolve an NPM specifier to the runtime module URL it will load: - - ```js - import { resolve } from 'oro:npm/module' - - const resolved = resolve('react', import.meta.url) - - console.log(resolved?.url) - console.log(resolved?.type) - ``` - """ - ), - "oro:os": md( - """ - Inspect the current platform and locate scratch space: - - ```js - import { arch, platform, tmpdir, cpus } from 'oro:os' - - console.log(platform(), arch()) - console.log(tmpdir()) - console.log(cpus().length) - ``` - """ - ), - "oro:path": md( - """ - Compose paths without branching on platform separators: - - ```js - import { resolve, join, dirname, basename } from 'oro:path' - - const absolute = resolve('dist', 'assets', 'logo.svg') - - console.log(absolute) - console.log(dirname(absolute)) - console.log(basename(join('/tmp', 'report.txt'))) - ``` - """ - ), - "oro:process": md( - """ - Inspect process state and schedule work on the next tick: - - ```js - import process, { env, hrtime, nextTick } from 'oro:process' - - console.log(env.PATH) - console.log(hrtime()) - - nextTick(() => { - console.log(process.pid) - }) - ``` - """ - ), - "oro:protocol-handlers": md( - """ - Ask the runtime which service worker is currently handling a custom scheme: - - ```js - import { getServiceWorker } from 'oro:protocol-handlers' - - const worker = await getServiceWorker({ scheme: 'npm' }) - - console.log(worker) - ``` - """ - ), - "oro:querystring": md( - """ - Parse and serialize query strings with the Node-compatible helpers: - - ```js - import { stringify, parse } from 'oro:querystring' - - const query = stringify({ q: 'oro runtime', page: 2 }) - - console.log(query) - console.log(parse(query)) - ``` - """ - ), - "oro:semver": md( - """ - Validate and compare versions before running upgrade logic: - - ```js - import { valid, compare, gt } from 'oro:semver' - - console.log(valid('1.4.0')) - console.log(compare('1.4.0', '1.3.9')) - console.log(gt('2.0.0', '1.9.0')) - ``` - """ - ), - "oro:service-worker": md( - """ - Use the runtime service-worker helpers from inside a service-worker scope: - - ```js - import { env } from 'oro:service-worker' - - if (env) { - self.addEventListener('fetch', (event) => { - event.respondWith(fetch(event.request)) - }) - } - ``` - """ - ), - "oro:shared-worker": md( - """ - Launch a shared worker and communicate through its shared port: - - ```js - import { SharedWorker } from 'oro:shared-worker' - - const worker = new SharedWorker(new URL('./shared.js', import.meta.url)) - - worker.port.start() - worker.port.postMessage({ type: 'warmup' }) - ``` - """ - ), - "oro:signal": md( - """ - Listen for runtime process signals using the signal helper surface: - - ```js - import signal from 'oro:signal' - - signal.addEventListener('SIGTERM', (event) => { - console.log(event.type) - }) - ``` - """ - ), - "oro:sqlite": md( - """ - Open an in-memory database and run a few synchronous queries: - - ```js - import { open, OPEN_CREATE, OPEN_MEMORY } from 'oro:sqlite' - - const db = open(':memory:', { flags: OPEN_CREATE | OPEN_MEMORY }) - - db.exec('create table notes (id integer primary key, title text)') - db.exec(\"insert into notes (title) values ('hello')\") - - console.log(db.query('select * from notes')) - ``` - """ - ), - "oro:stream": md( - """ - Connect streams together with the pipeline helper: - - ```js - import process from 'oro:process' - import { Readable, Writable, pipelinePromise } from 'oro:stream' - - const source = Readable.from(['oro', ' ', 'runtime']) - const sink = new Writable({ - write(chunk, _encoding, done) { - process.stdout.write(chunk) - done() - } - }) - - await pipelinePromise(source, sink) - ``` - """ - ), - "oro:string_decoder": md( - """ - Decode chunked UTF-8 data without splitting multibyte characters: - - ```js - import Buffer from 'oro:buffer' - import StringDecoder from 'oro:string_decoder' - - const decoder = new StringDecoder('utf8') - const chunks = [Buffer.from([0xe2, 0x82]), Buffer.from([0xac])] - - console.log(decoder.write(chunks[0])) - console.log(decoder.write(chunks[1])) - ``` - """ - ), - "oro:tar": md( - """ - Build a tar archive in memory and inspect its entries: - - ```js - import Buffer from 'oro:buffer' - import { createInMemory } from 'oro:tar' - - const archive = await createInMemory() - - await archive.append({ path: 'hello.txt' }, Buffer.from('hello')) - await archive.finalize() - - console.log(await archive.entries()) - ``` - """ - ), - "oro:tcp": md( - """ - Open a lower-level TCP server when you want more direct socket control: - - ```js - import { createServer } from 'oro:tcp' - - const server = createServer((socket) => { - socket.write(Buffer.from('connected')) - }) - - server.listen({ port: 5050, host: '127.0.0.1' }) - ``` - """ - ), - "oro:test": md( - """ - Write runtime tests with the built-in test harness: - - ```js - import test from 'oro:test' - - test('querystring roundtrip', async (t) => { - t.equal('a=1', new URLSearchParams({ a: '1' }).toString()) - }) - ``` - """ - ), - "oro:timers": md( - """ - Sleep without writing your own timeout wrapper: - - ```js - import { sleep } from 'oro:timers' - - await sleep(250) - - console.log('quarter second elapsed') - ``` - """ - ), - "oro:tls": md( - """ - Configure TLS pins before opening pinned connections: - - ```js - import { setTlsPins, getTlsPins, clearTlsPins } from 'oro:tls' - - await setTlsPins('sha256/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=') - console.log(await getTlsPins()) - await clearTlsPins() - ``` - """ - ), - "oro:toml": md( - """ - Parse configuration and emit updated TOML back to disk: - - ```js - import { parse, stringify } from 'oro:toml' - - const config = parse('name = \"oro\"\\nport = 8080') - config.debug = true - - console.log(stringify(config)) - ``` - """ - ), - "oro:tty": md( - """ - Detect interactive terminals before enabling richer output: - - ```js - import { isatty } from 'oro:tty' - - console.log(isatty(0)) - console.log(isatty(1)) - ``` - """ - ), - "oro:url": md( - """ - Use the WHATWG URL implementation for robust URL parsing: - - ```js - import URL from 'oro:url' - - const url = new URL('https://oro.computer/docs?project=runtime') - - console.log(url.pathname) - console.log(url.searchParams.get('project')) - ``` - """ - ), - "oro:usb": md( - """ - Install the runtime WebUSB surface and request a device in window contexts: - - ```js - import { installNavigatorUSB } from 'oro:usb' - - const usb = installNavigatorUSB() - - if (usb) { - console.log(await usb.getDevices()) - // const device = await usb.requestDevice({ filters: [{ vendorId: 0x1209 }] }) - } - ``` - """ - ), - "oro:util": md( - """ - Use the runtime debug helper and common type predicates: - - ```js - import { debug, isTypedArray } from 'oro:util' - - const log = debug('docs') - const payload = new Uint8Array([1, 2, 3]) - - log('payload ready') - console.log(isTypedArray(payload)) - ``` - """ - ), - "oro:vm": md( - """ - Run a script inside an isolated context object: - - ```js - import { runInContext } from 'oro:vm' - - const context = { answer: 41 } - const result = await runInContext('answer + 1', context) - - console.log(result) - ``` - """ - ), - "oro:worker": md( - """ - Launch a dedicated worker through the runtime worker surface: - - ```js - import Worker from 'oro:worker' - - const worker = new Worker(new URL('./worker.js', import.meta.url), { - workerData: { job: 'thumbnail' } - }) - - worker.postMessage({ type: 'start' }) - ``` - """ - ), - "oro:worker_threads": md( - """ - Pass environment data into worker threads and communicate with message channels: - - ```js - import { MessageChannel, setEnvironmentData } from 'oro:worker_threads' - - setEnvironmentData('traceId', 'req-42') - - const { port1, port2 } = new MessageChannel() - port1.onmessage = (event) => console.log(event.data) - port2.postMessage({ ok: true }) - ``` - """ - ), - "oro:xpc": md( - """ - Open an XPC connection, send a request, and close it cleanly: - - ```js - import { availability, connect } from 'oro:xpc' - - const state = await availability() - - if (state.available) { - const connection = await connect({ service: 'com.example.agent' }) - await connection.sendAndForget({ type: 'ping' }) - await connection.close() - } - ``` - """ - ), - "oro:zlib": md( - """ - Compress and decompress buffers with the promise-based helpers: - - ```js - import { gzip, gunzip } from 'oro:zlib' - - const compressed = await gzip('hello runtime') - const restored = await gunzip(compressed) - - console.log(restored.toString()) - ``` - """ - ), -} - - -DEFAULT_SEE_ALSO: tuple[tuple[str, str], ...] = ( - ("JavaScript APIs overview", "javascript/overview"), - ("All module specifiers", "javascript/all-modules"), -) - - -GUIDE_REFS_BY_FAMILY: dict[str, tuple[tuple[str, str], ...]] = { - "oro:application": ( - ("Windows and messaging", "guides/windows-and-messaging"), - ("Lifecycle hooks and deep links", "guides/lifecycle-hooks-and-deep-links"), - ("Desktop integrations", "guides/desktop-integrations"), - ), - "oro:assert": ( - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:cdp": ( - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:child_process": ( - ("Configure your runtime project", "guides/configure-your-runtime-project"), - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:clipboard": ( - ("Desktop integrations", "guides/desktop-integrations"), - ), - "oro:console": ( - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:crypto": ( - ("Secure storage and sessions", "guides/secure-storage-and-sessions"), - ), - "oro:diagnostics": ( - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:errors": ( - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:extension": ( - ("Native extensions", "guides/native-extensions"), - ), - "oro:fetch": ( - ("Calling HTTP APIs", "guides/calling-http-apis"), - ("Offline-first with service workers", "guides/offline-first-with-service-workers"), - ), - "oro:fs": ( - ("Files and sandboxing", "guides/files-and-sandboxing"), - ("Project layout", "guides/project-layout"), - ), - "oro:hooks": ( - ("Lifecycle hooks and deep links", "guides/lifecycle-hooks-and-deep-links"), - ("Windows and messaging", "guides/windows-and-messaging"), - ), - "oro:http": ( - ("Calling HTTP APIs", "guides/calling-http-apis"), - ), - "oro:https": ( - ("Calling HTTP APIs", "guides/calling-http-apis"), - ), - "oro:ipc": ( - ("Native extensions", "guides/native-extensions"), - ("Windows and messaging", "guides/windows-and-messaging"), - ), - "oro:location": ( - ("Lifecycle hooks and deep links", "guides/lifecycle-hooks-and-deep-links"), - ), - "oro:mcp": ( - ("MCP and agent automation", "guides/mcp-and-agent-automation"), - ), - "oro:navigation": ( - ("Lifecycle hooks and deep links", "guides/lifecycle-hooks-and-deep-links"), - ("Windows and messaging", "guides/windows-and-messaging"), - ), - "oro:notification": ( - ("Desktop integrations", "guides/desktop-integrations"), - ("Lifecycle hooks and deep links", "guides/lifecycle-hooks-and-deep-links"), - ), - "oro:path": ( - ("Files and sandboxing", "guides/files-and-sandboxing"), - ("Project layout", "guides/project-layout"), - ), - "oro:process": ( - ("Configure your runtime project", "guides/configure-your-runtime-project"), - ), - "oro:protocol-handlers": ( - ("Custom protocols and routing", "guides/custom-protocols-and-routing"), - ("Offline-first with service workers", "guides/offline-first-with-service-workers"), - ), - "oro:secure-storage": ( - ("Secure storage and sessions", "guides/secure-storage-and-sessions"), - ), - "oro:service-worker": ( - ("Offline-first with service workers", "guides/offline-first-with-service-workers"), - ("Custom protocols and routing", "guides/custom-protocols-and-routing"), - ), - "oro:shared-worker": ( - ("Shared workers across windows", "guides/shared-workers-across-windows"), - ("Windows and messaging", "guides/windows-and-messaging"), - ), - "oro:sqlite": ( - ("Local data with SQLite", "guides/local-data-with-sqlite"), - ("Files and sandboxing", "guides/files-and-sandboxing"), - ), - "oro:test": ( - ("Testing and diagnostics", "guides/testing-and-diagnostics"), - ), - "oro:worker": ( - ("Worker threads for heavy work", "guides/worker-threads-for-heavy-work"), - ("Shared workers across windows", "guides/shared-workers-across-windows"), - ), - "oro:worker_threads": ( - ("Worker threads for heavy work", "guides/worker-threads-for-heavy-work"), - ), - "oro:window": ( - ("Windows and messaging", "guides/windows-and-messaging"), - ("Desktop integrations", "guides/desktop-integrations"), - ), -} diff --git a/sage/tools/audit-site.py b/sage/tools/audit-site.py deleted file mode 100644 index 3ca6450f..00000000 --- a/sage/tools/audit-site.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import sys -import os -from pathlib import Path - - -SITE_ROOT = Path(__file__).resolve().parents[2] -SAGE_ROOT = Path(os.environ.get("ORO_SITE_OUTPUT", str(SITE_ROOT / "public"))) / "sage" -REPO_ROOT = SAGE_ROOT.parent - -sys.path.insert(0, str(SITE_ROOT / "tools")) - -from site_audit_common import SiteAuditConfig, run_site_audit - - -def main() -> int: - return run_site_audit( - SiteAuditConfig( - project_root=SAGE_ROOT, - docs_source=SAGE_ROOT / "docs" / "source", - docs_index=SAGE_ROOT / "docs" / "index.json", - missing_index_message="Missing Sage docs/index.json; run build scripts first.", - success_message="OK: sage site audit passed", - ) - ) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/silk/tools/audit-site.py b/silk/tools/audit-site.py deleted file mode 100644 index 64228e25..00000000 --- a/silk/tools/audit-site.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import re -import sys -import os -from pathlib import Path - - -SITE_ROOT = Path(__file__).resolve().parents[2] -SILK_ROOT = Path(os.environ.get("ORO_SITE_OUTPUT", str(SITE_ROOT / "public"))) / "silk" -REPO_ROOT = SILK_ROOT.parent - -DOCS_SOURCE = SILK_ROOT / "docs" / "source" -WIKI_SOURCE = SILK_ROOT / "wiki" / "source" - -DOCS_INDEX = SILK_ROOT / "docs" / "index.json" -WIKI_INDEX = SILK_ROOT / "wiki" / "index.json" - -sys.path.insert(0, str(SITE_ROOT / "tools")) - -from site_audit_common import ( - Issue, - check_doclike_links_in_tree, - check_index_files_exist_at, - check_no_editorial_status_framing_in_tree, - check_no_raw_manpage_refs_in_tree, - check_no_raw_viewer_refs_in_tree, - check_p_links_in_tree, - load_ids, -) - - -def check_no_arena_identifiers(source_root: Path) -> list[Issue]: - issues: list[Issue] = [] - banned = re.compile( - r"\bconst\s+region\s+arena\b" - r"|\bexport\s+const\s+region\s+arena\b" - r"|\bwith\s+arena\b\s*\{" - r"|\bfrom\s+arena\b" - r"|\bglobal_arena\b", - flags=re.I, - ) - for md in source_root.rglob("*.md"): - text = md.read_text(encoding="utf-8") - if banned.search(text): - issues.append(Issue(md, "Found arena identifier in examples (use regions + neutral names).")) - return issues - - -def check_no_works_today_labels(source_root: Path) -> list[Issue]: - issues: list[Issue] = [] - banned = re.compile( - r"\bWorks today\b|\bWhat works today\b|\(Works today\)|Syntax\s*\(Selected\)", - flags=re.I, - ) - for md in source_root.rglob("*.md"): - text = md.read_text(encoding="utf-8") - if banned.search(text): - issues.append(Issue(md, "Found deprecated 'Works today' / 'Syntax (Selected)' labeling.")) - return issues - - -def check_spec_has_no_repo_internal_paths(spec_path: Path) -> list[Issue]: - issues: list[Issue] = [] - if not spec_path.exists(): - return issues - - text = spec_path.read_text(encoding="utf-8") - banned = re.compile( - r"\bchecker\.[A-Za-z0-9_]+\b" - r"|\bsrc/[A-Za-z0-9_./-]+" - r"|\bc-tests/" - r"|\btests/silk/", - flags=re.I, - ) - if banned.search(text): - issues.append(Issue(spec_path, "Spec contains repo-internal paths/names (remove or generalize).")) - return issues - - -def main() -> int: - if not DOCS_INDEX.exists() or not WIKI_INDEX.exists(): - print("Missing Silk index.json files; run build scripts first.", file=sys.stderr) - return 2 - - docs_ids = load_ids(DOCS_INDEX) - wiki_ids = load_ids(WIKI_INDEX) - ids_by_kind = {"docs": docs_ids, "wiki": wiki_ids} - source_roots = { - "docs": DOCS_SOURCE, - "wiki": WIKI_SOURCE, - "spec": DOCS_SOURCE / "spec", - } - - issues: list[Issue] = [] - issues += check_index_files_exist_at(DOCS_INDEX, DOCS_SOURCE) - issues += check_index_files_exist_at(WIKI_INDEX, WIKI_SOURCE) - issues += check_p_links_in_tree(DOCS_SOURCE, "docs", ids_by_kind, include_kind_in_message=True) - issues += check_p_links_in_tree(WIKI_SOURCE, "wiki", ids_by_kind, include_kind_in_message=True) - issues += check_doclike_links_in_tree(DOCS_SOURCE, REPO_ROOT, source_roots) - issues += check_doclike_links_in_tree(WIKI_SOURCE, REPO_ROOT, source_roots) - issues += check_no_arena_identifiers(DOCS_SOURCE) - issues += check_no_arena_identifiers(WIKI_SOURCE) - issues += check_no_works_today_labels(DOCS_SOURCE) - issues += check_no_works_today_labels(WIKI_SOURCE) - issues += check_no_raw_viewer_refs_in_tree(DOCS_SOURCE) - issues += check_no_raw_viewer_refs_in_tree(WIKI_SOURCE) - issues += check_no_raw_manpage_refs_in_tree(DOCS_SOURCE) - issues += check_no_raw_manpage_refs_in_tree(WIKI_SOURCE) - issues += check_no_editorial_status_framing_in_tree(DOCS_SOURCE) - issues += check_no_editorial_status_framing_in_tree(WIKI_SOURCE) - issues += check_spec_has_no_repo_internal_paths(DOCS_SOURCE / "spec" / "2026.md") - - if issues: - for issue in issues[:200]: - rel = issue.path.relative_to(REPO_ROOT) - print(f"{rel}: {issue.message}", file=sys.stderr) - if len(issues) > 200: - print(f"... and {len(issues) - 200} more", file=sys.stderr) - print(f"FAIL: {len(issues)} issues", file=sys.stderr) - return 1 - - print("OK: silk site audit passed") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/silk/tools/audit-stdlib-docs.py b/silk/tools/audit-stdlib-docs.py deleted file mode 100644 index f784f6d2..00000000 --- a/silk/tools/audit-stdlib-docs.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import os -import re -import sys -from dataclasses import dataclass -from pathlib import Path - - -SILK_ROOT = Path(__file__).resolve().parents[1] -STD_DOCS = Path(os.environ.get("ORO_SITE_OUTPUT", str(SILK_ROOT.parent / "public"))) / "silk" / "docs" / "source" / "std" - -ORDER = [ - "Description", - "Exported API", - "Examples", - "Considerations", - "Design goals", - "See also", -] - -BANNED_HEADINGS = ( - "API", - "Current API", - "Implemented API", - "Public API", - "High-Level API", - "Intended Surface", - "Current Surface", - "Current Grammar Coverage", - "Goals", - "Design Goals", - "Future Work", - "Future work", - "Follow-ups", - "Remaining Follow-Ups", - "Important Limitations", - "Current Limitations", - "Notes and Limitations", -) - -BANNED_HEADING_PATTERNS = ( - re.compile(r"\bInitial Design\b", flags=re.I), - re.compile(r"\bCurrent Scope\b", flags=re.I), - re.compile(r"\bScope \(Current\)\b", flags=re.I), - re.compile(r"\bMVP\b", flags=re.I), -) - -BANNED_STATUS_PATTERNS = ( - re.compile(r"active expansion", flags=re.I), - re.compile(r"current snapshot", flags=re.I), - re.compile(r"Partially implemented", flags=re.I), - re.compile(r"initial std wrapper", flags=re.I), - re.compile(r"expansion path", flags=re.I), - re.compile(r"\bMVP\b", flags=re.I), -) - - -@dataclass(frozen=True) -class Issue: - path: Path - message: str - - -def collect_h2s(path: Path) -> list[str]: - headings: list[str] = [] - for line in path.read_text(encoding="utf-8").splitlines(): - if line.startswith("## "): - headings.append(line[3:].strip()) - return headings - - -def check_status(path: Path, text: str) -> list[Issue]: - issues: list[Issue] = [] - for pattern in BANNED_STATUS_PATTERNS: - if pattern.search(text): - issues.append(Issue(path, f"Status/body uses banned transitional wording: {pattern.pattern}")) - return issues - - -def check_headings(path: Path, headings: list[str]) -> list[Issue]: - issues: list[Issue] = [] - - for heading in headings: - for banned in BANNED_HEADINGS: - if heading == banned: - issues.append(Issue(path, f"Banned heading: {heading}")) - for pattern in BANNED_HEADING_PATTERNS: - if pattern.search(heading): - issues.append(Issue(path, f"Banned heading pattern: {heading}")) - - positions = {heading: index for index, heading in enumerate(headings) if heading in ORDER} - last = -1 - seen: list[str] = [] - for heading in ORDER: - pos = positions.get(heading) - if pos is None: - continue - if pos < last: - issues.append( - Issue( - path, - f"Section order violation: {' -> '.join(seen + [heading])}", - ) - ) - break - last = pos - seen.append(heading) - - return issues - - -def main() -> int: - issues: list[Issue] = [] - - for path in sorted(STD_DOCS.glob("*.md")): - text = path.read_text(encoding="utf-8") - issues.extend(check_status(path, text)) - issues.extend(check_headings(path, collect_h2s(path))) - - if issues: - for issue in issues: - rel = issue.path.relative_to(SILK_ROOT.parent) - print(f"{rel}: {issue.message}") - return 1 - - print("Stdlib doc audit passed.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/silk/tools/sync-from-silk-docs.py b/silk/tools/sync-from-silk-docs.py deleted file mode 100644 index 5c6840a8..00000000 --- a/silk/tools/sync-from-silk-docs.py +++ /dev/null @@ -1,767 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import re -import shutil -import sys -import tempfile -from dataclasses import dataclass -from pathlib import Path -from typing import Callable - -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) -from importlib import import_module -source_pages = import_module("source-pages") - - -EXCLUDE_BASENAMES = { - # Arenas were removed from the user-facing surface; use regions instead. - "arenas.md", - "README.md", - "PLAN.md", - "STATUS.md", - "llms.txt", - "style-guide.md", - "TEMPLATE.md", - "_template-language.md", - "_template-stdlib.md", -} - - -KEEP_DOCS_PREFIXES = ( - "guides/", -) - - -KEEP_DOCS_FILES = { - # Website-owned landing/start. - "start.md", - "compiler/implementation-status.md", - # Website-owned, downstream-facing rewrites. - "usage/cli-examples.md", - "usage/howto-custom-stdlib-root.md", - "usage/howto-run-wasi-node.md", - "usage/editor-coc-nvim.md", - "usage/editor-ctags.md", - "usage/editor-vim.md", - "usage/editor-textmate.md", - "usage/getting-started.md", - "usage/github-linguist.md", - # Website-owned tutorial rewrites (avoid repo-build steps). - "usage/tutorials/01-first-program.md", - "usage/tutorials/02-structs-and-impls.md", - "usage/tutorials/03-arrays-and-slices.md", - "usage/tutorials/04-filesystem.md", - "usage/tutorials/05-concurrency.md", - "usage/tutorials/06-async-io-streams-abort.md", - "usage/tutorials/07-formal-silk.md", - # Website-owned copy edits to avoid repo-internal wording/refs. - "compiler/backend-wasm.md", - "compiler/testing-strategy.md", - "compiler/libsilk-quickstart.md", - "compiler/zig-api.md", -} - - -KEEP_WIKI_FILES = { - # Website-owned wiki landing/start. - "start.md", -} - - -@dataclass(frozen=True) -class SyncStats: - copied: int = 0 - skipped: int = 0 - deleted: int = 0 - - -def should_skip(rel: str, keep_files: set[str], keep_prefixes: tuple[str, ...]) -> bool: - name = Path(rel).name - if name in EXCLUDE_BASENAMES: - return True - if rel in keep_files: - return True - for prefix in keep_prefixes: - if rel.startswith(prefix): - return True - return False - - -def normalize_region_identifiers(markdown: str) -> str: - out = markdown - replacements = ( - (r"\bexport const region global_arena\b", "export const region global_region_buf"), - (r"\bconst region arena\b", "const region region_buf"), - (r"\bwith arena\b", "with region_buf"), - (r"\bfrom arena\b", "from region_buf"), - (r"\bglobal_arena\b", "global_region_buf"), - (r"\barena\[", "region_buf["), - (r"`arena`", "`region_buf`"), - (r"`global_arena`", "`global_region_buf`"), - ) - for pattern, replacement in replacements: - out = re.sub(pattern, replacement, out) - return out - - -def normalize_shared_markdown(markdown: str) -> str: - markdown = markdown.replace("https://oro.computer/silk/docs/?p=", "?p=") - markdown = markdown.replace("https://oro.computer/silk/wiki/?p=", "wiki/?p=") - markdown = markdown.replace("?p=docs/", "?p=") - markdown = markdown.replace("docs/?p=", "?p=") - markdown = markdown.replace("?p=wiki/", "wiki/?p=") - markdown = re.sub( - r"(?m)^(\s*//\s*)Works today\b", - r"\1Supported", - markdown, - ) - - markdown = re.sub(r"(?m)^(\s*#{2,6}\s+)Syntax\s+\(Selected\)\s*$", r"\1Syntax", markdown) - markdown = re.sub( - r"(?m)^(\s*#{2,6}\s+)(Example|Examples)\s+\(Works today\)(?::\s*(.+))?\s*$", - lambda m: f"{m.group(1)}{m.group(2)}" + (f": {m.group(3)}" if m.group(3) else ""), - markdown, - ) - markdown = re.sub( - r"(?m)^(\s*#{2,6}\s+)Works today(?::\s*(.+))?\s*$", - lambda m: f"{m.group(1)}Example" + (f": {m.group(2)}" if m.group(2) else ""), - markdown, - ) - markdown = re.sub( - r"(?m)^(\s*#{2,6}\s+)What works today(?:\s*\([^)]*\))?\s*$", - r"\1Current subset", - markdown, - ) - markdown = re.sub(r"(?m)^Works today\b", "Current subset", markdown) - markdown = markdown.replace("examples labeled “Works today”", "examples labeled “Example”") - markdown = normalize_region_identifiers(markdown) - return markdown - - -def normalize_user_facing_links(markdown: str) -> str: - def link_label(target: str) -> str: - stem = target.rsplit("/", 1)[-1] - return stem.replace("-", " ").replace("_", " ") - - def doc_link_target(target: str) -> str | None: - target = target.removesuffix(".md") - for prefix in ("docs/", "wiki/"): - if target.startswith(prefix): - target = target[len(prefix):] - target = { - "language/README": "start", - "usage/README": "start", - "usage/tutorials/README": "start", - "std/runtime-event-loop": "std/runtime", - }.get(target, target) - if target in {"std/foo", "std/foo-bar", "man/my-app.7"}: - return None - return target - - def normalize_line(line: str) -> str: - line = re.sub(r"`?\?p=\.\.\.`?", "docs links", line) - line = re.sub( - r"`\?p=(?P[a-zA-Z0-9_./-]+)`", - lambda m: f"[{link_label(m.group('target'))}](?p={m.group('target')})", - line, - ) - - def replace_raw_viewer_ref(match: re.Match[str]) -> str: - prefix = match.group("prefix") or "" - target = doc_link_target(match.group("target")) - if prefix == "[" or target is None: - return match.group(0) - return f"{prefix}[{link_label(target)}](?p={target})" - - line = re.sub( - r"(?P^|[\s:;,(])(?[a-zA-Z0-9_./-]+)", - replace_raw_viewer_ref, - line, - ) - - def replace_man_ref(match: re.Match[str]) -> str: - prefix = match.group("prefix") or "" - name = match.group("name") - section = match.group("section") - if name.startswith("std::"): - target = name.replace("::", "/") - return f"{prefix}[`{name}({section})`](?p={target})" - return f"{prefix}[`{name}({section})`](?p=man/{name}.{section})" - - line = re.sub( - r"(?P^|[\s:;,(])(?[A-Za-z0-9_:+.-]+)`\s+\((?P
[137])\)", - replace_man_ref, - line, - ) - - def replace_markdown_docs_link(match: re.Match[str]) -> str: - target = doc_link_target(match.group("target")) - if target is None: - return match.group(0) - return f"[{link_label(target)}](?p={target})" - - line = re.sub( - r"\[`?docs/(?P[a-zA-Z0-9_./-]+\.md)`?\]\([^)]+\)", - replace_markdown_docs_link, - line, - ) - - def replace_docs_path(match: re.Match[str]) -> str: - prefix = match.group("prefix") or "" - target = doc_link_target(match.group("target")) - if target is None: - return match.group(0) - return f"{prefix}[{link_label(target)}](?p={target})" - - line = re.sub( - r"(?P^|[\s:;,(])`?docs/(?P[a-zA-Z0-9_./-]+\.md)`?", - replace_docs_path, - line, - ) - return line - - out_lines: list[str] = [] - in_code = False - for raw in markdown.splitlines(): - stripped = raw.lstrip() - if stripped.startswith("```") or stripped.startswith("~~~"): - in_code = not in_code - out_lines.append(raw.rstrip()) - continue - out_lines.append(raw.rstrip() if in_code else normalize_line(raw.rstrip())) - return "\n".join(out_lines) + ("\n" if markdown.endswith("\n") else "") - - -def normalize_editorial_framing(markdown: str) -> str: - replacements = ( - (r"\bStatus:\s*\*\*([^*]+)\*\*\.\s*", ""), - (r"\bStatus:\s*\*\*([^*]+)\*\*\s*", ""), - (r"\bStatus:\s*", ""), - (r"\bImplementation status\s*\([^)]*\)\s*:\s*", ""), - (r"\bImplementation status\s*:\s*", ""), - (r"Returning readers typically want the “Implementation Status” section near the top", "Returning readers typically want the notes near the top"), - (r"see “Implementation Status” below", "see the notes below"), - (r"Implementation Status” sections", "notes sections"), - (r"\bCurrent supported contexts include\b", "Supported contexts include"), - (r"\bcurrent supported\b", "supported"), - (r"\bCurrent compiler subset restriction\b", "Restriction"), - (r"\bthe current compiler subset\b", "Silk currently"), - (r"\bThe current compiler subset\b", "Silk currently"), - (r"\bcurrent compiler subset\b", "current implementation"), - (r"\bCurrent subset\b", "Supported forms"), - (r"\bcurrent subset\b", "supported forms"), - (r"\bInitial implementation target\b", "Implementation target"), - (r"\binitial implementation target\b", "implementation target"), - (r"\binitial implementation\b", "implementation"), - (r"\bInitial implementation\b", "Implementation"), - (r"\bImplemented-subset notes\b", "Notes"), - (r"\bImplemented-subset details\b", "Reference details"), - (r"\bImplemented-subset\b", "Reference"), - (r"\bImplemented subset notes\b", "Notes"), - (r"\bimplemented subset notes\b", "notes"), - (r"\bImplemented subset\b", "Supported forms"), - (r"\bimplemented subset\b", "supported forms"), - (r"\bImplemented Subset\b", "Supported Forms"), - (r"\bInitial Implemented Subset\b", "Supported Forms"), - (r"\bCurrent Implemented Subset\b", "Supported Forms"), - (r"\bexamples that exercise the supported forms\b", "examples"), - (r"\bExamples that exercise the supported forms\b", "Examples"), - (r"\bactive expansion\b", "current module surface"), - (r"\bPartially implemented\b", "Implemented"), - (r"\bpartially implemented\b", "implemented"), - (r"\bin progress\b", "available"), - (r"\bIn progress\b", "Available"), - ) - - out_lines: list[str] = [] - in_code = False - for raw in markdown.splitlines(): - stripped = raw.lstrip() - if stripped.startswith("```") or stripped.startswith("~~~"): - in_code = not in_code - out_lines.append(raw.rstrip()) - continue - if in_code: - out_lines.append(raw.rstrip()) - continue - - line = raw - heading = re.match(r"^(?P\s*#{1,6}\s+)(?P.+?)\s*$", line) - if heading: - title = heading.group("title").strip() - title = re.sub( - r"\s+\((?:Initial\s+)?(?:Current\s+)?(?:Implemented|supported|compiler|initial|selected|planned|current|implementation status)[^)]*\)", - "", - title, - flags=re.I, - ) - title = re.sub(r"\s+\(Planned[^)]*\)", "", title, flags=re.I) - if re.fullmatch(r"Status(?:\s*\([^)]*\))?|Implementation Status(?:\s*\([^)]*\))?|Status and Future Work|Future Work|Follow-ups", title, flags=re.I): - title = "Notes" - elif re.fullmatch(r"Notes and Limitations", title, flags=re.I): - title = "Considerations" - elif re.fullmatch(r"Current API(?:\s*\([^)]*\))?|Implemented API|Public API|API\s*\([^)]*\)|API\s*\(current\)", title, flags=re.I): - title = "Exported API" - elif re.fullmatch(r"Current Implemented Subset|Implemented Subset|Implemented subset", title, flags=re.I): - title = "Notes" - title = re.sub(r"\s{2,}", " ", title).strip() - line = f"{heading.group('prefix')}{title}" - - for pattern, replacement in replacements: - line = re.sub(pattern, replacement, line, flags=re.I) - line = re.sub(r"\s{2,}", " ", line).rstrip() - out_lines.append(line) - - return "\n".join(out_lines) + ("\n" if markdown.endswith("\n") else "") - - -def normalize_trailing_newlines(markdown: str) -> str: - return markdown.rstrip() + "\n" - - -def normalize_headings_for_context(path: Path, markdown: str) -> str: - markdown = markdown.replace("`docs/language/mutability.md`", "the mutability docs") - markdown = markdown.replace("`docs/language/literals-duration.md`", "the duration literal docs") - markdown = markdown.replace("?p=std/runtime-event-loop", "?p=std/runtime") - - if "/wiki/source/" in path.as_posix(): - markdown = re.sub(r"\]\(\?p=(compiler|man|usage|guides)/", r"](../docs/?p=\1/", markdown) - - if path.parent.name == "std": - markdown = re.sub(r"(?m)^## API$", "## Exported API", markdown) - markdown = re.sub(r"(?m)^## High-Level API$", "## Exported API", markdown) - markdown = re.sub(r"(?m)^## Intended Surface$", "## Exported API", markdown) - markdown = re.sub(r"(?m)^## Current Surface$", "## Exported API", markdown) - markdown = re.sub(r"(?m)^## Current Grammar Coverage$", "## Grammar coverage", markdown) - markdown = re.sub(r"(?m)^## Goals$", "## Design goals", markdown) - markdown = re.sub(r"(?m)^## Design Goals$", "## Design goals", markdown) - markdown = re.sub(r"(?m)^## Important Limitations$", "## Considerations", markdown) - markdown = re.sub(r"(?m)^## Remaining Follow-Ups$", "## Considerations", markdown) - markdown = re.sub(r"(?m)^## Implemented (`std::interfaces` surface)$", r"## \1", markdown) - markdown = markdown.replace("current API surface", "exported API surface") - markdown = markdown.replace("API surface (current):", "API surface:") - markdown = move_section_to_end(markdown, "Design goals") - - if path.parent.name == "language": - markdown = re.sub(r"(?m)^## Goals$", "## Semantics", markdown) - markdown = re.sub(r"(?m)^## Design Goals$", "## Model", markdown) - markdown = re.sub(r"(?m)^### Important Limitations$", "### Considerations", markdown) - - if path.name == "flow-for.md": - markdown = markdown.replace( - "element.\n\n: integer range iteration", - "element.\n\nSupported forms include integer range iteration", - ) - markdown = re.sub( - r"\n## Semantics\n\n- Provide a readable, structured loop construct for iteration\.\n- Avoid .+? hidden allocation\.\n\n## Surface Syntax", - "\n## Surface Syntax", - markdown, - flags=re.S, - ) - - return markdown - - -def sanitize_wiki_markdown(rel: str, markdown: str) -> str: - """ - The upstream Silk wiki is written for repo contributors and sometimes - references internal tracker files (STATUS.md, PLAN.md) that don't exist (or - aren't meaningful) on the public website. - - This function keeps the useful parts of those lines while removing the - internal-only references. - """ - - markdown = normalize_shared_markdown(markdown) - - out_lines: list[str] = [] - - drop_whole_line = re.compile( - r"^\s*[-*+]\s*(End-to-end support snapshot|Implemented-subset notes)\s*:\s*`?(STATUS|PLAN)\.md`?\s*$", - flags=re.I, - ) - - in_code = False - - for raw in markdown.splitlines(): - trimmed = raw.lstrip() - if trimmed.startswith("```"): - in_code = not in_code - out_lines.append(raw.rstrip()) - continue - - if in_code: - # Never rewrite inside fenced code blocks; preserve indentation and - # exact spelling. - out_lines.append(raw.rstrip()) - continue - - line = raw - - if drop_whole_line.match(line): - continue - - # Rewrite a few common wiki-only phrasings. - line = re.sub(r"^(\s*[-*+]\s+)Relevant fixtures:\s*", r"\1Fixtures: ", line, flags=re.I) - line = re.sub( - r"^\s*Status:\s*implemented for the current front-end \+\s*native backend subset\.\s*$", - "Implemented in the reference compiler (front-end + native backend subset).", - line, - flags=re.I, - ) - - # Remove internal tracker refs while preserving surrounding prose. - for basename in ("STATUS.md", "PLAN.md"): - # Common: "... and `STATUS.md`" - line = re.sub(rf"\s+(?:and|&)\s+`?{re.escape(basename)}`?\s*$", "", line, flags=re.I) - # Strip any remaining mention. - line = re.sub(rf"`?{re.escape(basename)}`?", "", line, flags=re.I) - - # If we removed a reference, clean up dangling conjunctions. - line = re.sub(r"\s+(?:and|&)\s*$", "", line, flags=re.I) - - # Tidy up extra spaces introduced by stripping (but preserve indentation). - leading = re.match(r"^\s*", line).group(0) - body = line[len(leading) :] - body = re.sub(r"[ \t]{2,}", " ", body) - line = (leading + body).rstrip() - - # Drop empty bullets like "- Details:" after stripping. - if re.match(r"^\s*[-*+]\s*[^A-Za-z0-9`]*\s*$", line): - continue - - out_lines.append(line) - - cleaned = "\n".join(out_lines) + ("\n" if markdown.endswith("\n") else "") - cleaned = normalize_editorial_framing(cleaned) - cleaned = normalize_user_facing_links(cleaned) - return normalize_trailing_newlines(cleaned) - - -def sanitize_docs_markdown(rel: str, markdown: str) -> str: - """ - The upstream Silk docs are written for the Silk compiler repository and may - use ambiguous phrasing like "this repository" or project-internal jargon - like "repo dependency workflow". When we sync into the website, we rewrite - a few phrases to be clearer to downstream readers. - """ - - def drop_named_section(md: str, heading: str) -> str: - target = heading.strip().lower() - out_lines: list[str] = [] - in_code = False - skip_level: int | None = None - - for raw in md.splitlines(): - trimmed = raw.lstrip() - if trimmed.startswith("```"): - in_code = not in_code - if skip_level is None: - out_lines.append(raw.rstrip()) - continue - - if not in_code and skip_level is not None: - heading_match = re.match(r"^(#{1,6})\s+(.+?)\s*$", raw) - if not heading_match: - continue - level = len(heading_match.group(1)) - if level > skip_level: - continue - skip_level = None - - if not in_code: - heading_match = re.match(r"^(#{1,6})\s+(.+?)\s*$", raw) - if heading_match and heading_match.group(2).strip().lower() == target: - skip_level = len(heading_match.group(1)) - continue - - if skip_level is None: - out_lines.append(raw.rstrip()) - - return "\n".join(out_lines) + ("\n" if md.endswith("\n") else "") - - def rewrite_outside_inline_code(text: str) -> str: - out = text - - # Make references explicit. - out = re.sub(r"\bThe repository\b", "The Silk compiler repository", out) - out = re.sub(r"\bthe repository\b", "the Silk compiler repository", out) - out = re.sub(r"\bThis repository\b", "The Silk compiler repository", out) - out = re.sub(r"\bthis repository\b", "the Silk compiler repository", out) - - # Prefer a clearer name for the dependency build flow. - out = re.sub( - r"\brepo dependency workflow\b", - "Silk compiler repository’s vendored dependency workflow", - out, - flags=re.I, - ) - out = re.sub(r"\bwhat works today\b", "current implementation notes", out, flags=re.I) - - return out - - def sanitize_std_markdown(text: str) -> str: - out = text - out = re.sub(r"(?m)^Status:\s*\*\*[^*]+\*\*\.\s*", "", out) - out = re.sub(r"(?m)^Status:\s*\*\*[^*]+\*\*\.\s*$", "", out) - out = re.sub(r"(?m)^##\s+Current API\s*$", "## Exported API", out) - out = re.sub(r"(?m)^##\s+Implemented API\s*$", "## Exported API", out) - out = re.sub(r"(?m)^##\s+Public API\s*$", "## Exported API", out) - out = re.sub( - r"(?m)^##\s+(Future Work|Future work|Follow-ups|Current Limitations|Notes and Limitations)\s*$", - "## Considerations", - out, - ) - out = re.sub(r"(?m)^##\s+Current Scope\s*$", "## Considerations", out) - out = re.sub(r"(?m)^##\s+(.+?)\s+\((?:Initial Design|MVP|Current)\)\s*$", r"## \1", out) - out = re.sub(r"active expansion", "current module surface", out, flags=re.I) - out = re.sub(r"current snapshot", "current implementation", out, flags=re.I) - out = re.sub(r"Partially implemented", "Implemented", out, flags=re.I) - out = re.sub(r"initial std wrapper", "stdlib wrapper", out, flags=re.I) - out = re.sub(r"\bMVP\b", "baseline", out) - return out - - def sanitize_spec_markdown(text: str) -> str: - out = drop_named_section(text, "Silk Proposal Process (TC39-Inspired)") - out = out.replace("// Works today", "// Supported") - out = out.replace("checker.checkModuleSetWithImports", "the module-set import helper") - out = re.sub( - r"CLI output:\s*`+silk` CLI`?\s+and\s+`+silk`\s+\(1\)\s+—\s+Silk Language Compiler`?\s+\(`silk --version`\)", - "CLI output: the `silk` CLI and [`silk(1)`](?p=man/silk.1) (`silk --version`)", - out, - ) - return out - - markdown = normalize_shared_markdown(markdown) - markdown = markdown.replace( - "`STATUS.md`", - "[implementation status](?p=compiler/implementation-status)", - ) - markdown = markdown.replace( - "[implementation status](?p=compiler/implementation-status) (implementation status)", - "[implementation status](?p=compiler/implementation-status)", - ) - markdown = re.sub( - r"\s*\(tracked[^)]*`PLAN\.md`\)", - "", - markdown, - flags=re.I, - ) - markdown = drop_named_section(markdown, "Arenas") - markdown = drop_named_section(markdown, "Tests") - if rel.startswith("std/"): - markdown = sanitize_std_markdown(markdown) - if rel == "spec/2026.md": - markdown = sanitize_spec_markdown(markdown) - if rel == "compiler/vendored-deps.md": - markdown = markdown.replace( - "# Built-In Dependencies\n", - "# Built-In Dependencies (Legacy Link)\n", - 1, - ) - - out_lines: list[str] = [] - in_code = False - - for raw in markdown.splitlines(): - trimmed = raw.lstrip() - if trimmed.startswith("```"): - in_code = not in_code - out_lines.append(raw.rstrip()) - continue - - if in_code: - out_lines.append(raw.rstrip()) - continue - - # Preserve inline-code spans (single-backtick) while rewriting prose. - parts = raw.split("`") - for i in range(0, len(parts), 2): - parts[i] = rewrite_outside_inline_code(parts[i]) - out_lines.append("`".join(parts).rstrip()) - - cleaned = "\n".join(out_lines) + ("\n" if markdown.endswith("\n") else "") - cleaned = normalize_editorial_framing(cleaned) - cleaned = normalize_user_facing_links(cleaned) - return normalize_trailing_newlines(cleaned) - - - -def move_section_to_end(markdown: str, heading: str) -> str: - lines = markdown.splitlines() - section: list[str] = [] - out: list[str] = [] - in_code = False - capture = False - capture_level = 0 - - for raw in lines: - stripped = raw.lstrip() - if stripped.startswith("```") or stripped.startswith("~~~"): - in_code = not in_code - - if not in_code: - m = re.match(r"^(#{1,6})\s+(.+?)\s*$", raw) - if m: - level = len(m.group(1)) - title = m.group(2).strip() - if capture and level <= capture_level: - capture = False - if not capture and title.lower() == heading.lower(): - capture = True - capture_level = level - - if capture: - section.append(raw.rstrip()) - else: - out.append(raw.rstrip()) - - if not section: - return markdown - while out and not out[-1].strip(): - out.pop() - while section and not section[-1].strip(): - section.pop() - return "\n".join(out + ["", *section]) + "\n" - - -def postprocess_tree(root: Path, *, keep_files: set[str] = frozenset(), keep_prefixes: tuple[str, ...] = ()) -> None: - for path in sorted(root.rglob("*.md")): - if should_skip(path.relative_to(root).as_posix(), keep_files, keep_prefixes): - continue - text = path.read_text(encoding="utf-8") - def normalize(body): - body = normalize_editorial_framing(body) - body = normalize_user_facing_links(body) - body = normalize_headings_for_context(path, body) - return normalize_trailing_newlines(body) - next_text = source_pages.preserve_fences(text, normalize) - if next_text != text: - path.write_text(next_text, encoding="utf-8") - - -def sync_tree( - src_root: Path, - dst_root: Path, - *, - keep_files: set[str], - keep_prefixes: tuple[str, ...], - sanitize: Callable[[str, str], str] | None = None, -) -> SyncStats: - copied: set[str] = set() - skipped = 0 - deleted = 0 - - for path in sorted(src_root.rglob("*")): - if not path.is_file(): - continue - if path.suffix not in (".md", ".txt"): - continue - - rel = path.relative_to(src_root).as_posix() - if should_skip(rel, keep_files, keep_prefixes): - skipped += 1 - continue - - dst = dst_root / rel - dst.parent.mkdir(parents=True, exist_ok=True) - - if sanitize and path.suffix == ".md": - dst.write_text(sanitize(rel, path.read_text(encoding="utf-8")), encoding="utf-8") - else: - shutil.copyfile(path, dst) - copied.add(rel) - - # Prune any previously-synced files that no longer exist upstream. - for path in sorted(dst_root.rglob("*")): - if not path.is_file(): - continue - if path.suffix not in (".md", ".txt"): - continue - - if path.name in EXCLUDE_BASENAMES: - path.unlink() - deleted += 1 - continue - - rel = path.relative_to(dst_root).as_posix() - if should_skip(rel, keep_files, keep_prefixes): - continue - if rel in copied: - continue - path.unlink() - deleted += 1 - - return SyncStats(copied=len(copied), skipped=skipped, deleted=deleted) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Sync Silk docs/wiki from the repo (silk/docs) into the website copies (website/silk)." - ) - parser.add_argument( - "--repo-root", - type=Path, - default=None, - help="Path to repo root (auto-detected by default).", - ) - parser.add_argument("--silk-repo", type=Path, help="Path to the upstream Silk checkout.") - args = parser.parse_args() - - site = Path(__file__).resolve().parents[2] - silk_repo = args.silk_repo or ((args.repo_root / "silk") if args.repo_root else site.parent / "silk") - src_docs = silk_repo.resolve() / "docs" - src_wiki = src_docs / "wiki" - - staging = tempfile.TemporaryDirectory(prefix="oro-silk-import-") - dst_docs = Path(staging.name) / "docs" - dst_wiki = Path(staging.name) / "wiki" - source_pages.stage_collection(site, "silk", dst_docs) - source_pages.stage_collection(site, "silkWiki", dst_wiki) - - if not src_docs.exists(): - raise SystemExit(f"Missing source docs at {src_docs}") - - # Docs: copy everything except wiki/ subtree. - docs_stats = sync_tree( - src_docs, - dst_docs, - keep_files=KEEP_DOCS_FILES, - keep_prefixes=KEEP_DOCS_PREFIXES + ("wiki/",), - sanitize=lambda rel, text: source_pages.preserve_fences(text, lambda body: sanitize_docs_markdown(rel, body)), - ) - - # Wiki: copy everything under docs/wiki into website wiki source. - if src_wiki.exists(): - wiki_stats = sync_tree( - src_wiki, - dst_wiki, - keep_files=KEEP_WIKI_FILES, - keep_prefixes=(), - sanitize=lambda rel, text: source_pages.preserve_fences(text, lambda body: sanitize_wiki_markdown(rel, body)), - ) - else: - wiki_stats = SyncStats() - - postprocess_tree(dst_docs, keep_files=KEEP_DOCS_FILES, keep_prefixes=KEEP_DOCS_PREFIXES) - postprocess_tree(dst_wiki, keep_files=KEEP_WIKI_FILES) - - source_pages.import_collection(site, "silk", dst_docs) - if src_wiki.exists(): - source_pages.import_collection(site, "silkWiki", dst_wiki) - staging.cleanup() - - print("Synced Silk docs to website:") - print( - f"- Docs copied: {docs_stats.copied} (skipped: {docs_stats.skipped}, deleted: {docs_stats.deleted})" - ) - print( - f"- Wiki copied: {wiki_stats.copied} (skipped: {wiki_stats.skipped}, deleted: {wiki_stats.deleted})" - ) - - -if __name__ == "__main__": - main() diff --git a/silk/tools/sync-from-silk-docs.ts b/silk/tools/sync-from-silk-docs.ts new file mode 100644 index 00000000..17a6eaba --- /dev/null +++ b/silk/tools/sync-from-silk-docs.ts @@ -0,0 +1,88 @@ +#!/usr/bin/env node +import { access, copyFile, mkdir, mkdtemp, rm, unlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { basename, dirname, extname, join, relative, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { parseArgs } from 'node:util' +import { files, readText, preserveFences, stageCollection, importCollection } from '../../tools/source-pages.ts' +import { EXCLUDE_BASENAMES, KEEP_DOCS_FILES, KEEP_DOCS_PREFIXES, KEEP_WIKI_FILES } from '../../tools/silk-ingestion-data.ts' +import { sanitizeDocsMarkdown, sanitizeWikiMarkdown, normalizeEditorialFraming, normalizeUserFacingLinks, normalizeHeadingsForContext, normalizeTrailingNewlines } from '../../tools/silk-editorial.ts' +export { EXCLUDE_BASENAMES, KEEP_DOCS_FILES, KEEP_DOCS_PREFIXES, KEEP_WIKI_FILES } +export * from '../../tools/silk-editorial.ts' +export interface Ownership { keepFiles?: readonly string[]; keepPrefixes?: readonly string[] } +export interface SyncStats { copied: number; skipped: number; deleted: number } +export function shouldSkip(rel: string, { keepFiles = [], keepPrefixes = [] }: Ownership): boolean { + return EXCLUDE_BASENAMES.includes(basename(rel)) || keepFiles.includes(rel) || keepPrefixes.some(prefix => rel.startsWith(prefix)) +} +export async function syncTree(src: string, dst: string, options: Ownership & { sanitize?: (rel: string, text: string) => string }): Promise<SyncStats> { + const copied = new Set<string>() + let skipped = 0, deleted = 0 + for (const path of await files(src)) { + if (!['.md', '.txt'].includes(extname(path))) continue + const rel = relative(src, path) + if (shouldSkip(rel, options)) { skipped++; continue } + const dest = join(dst, rel) + await mkdir(dirname(dest), { recursive: true }) + if (options.sanitize && extname(path) === '.md') await writeFile(dest, options.sanitize(rel, await readText(path))) + else await copyFile(path, dest) + copied.add(rel) + } + for (const path of await files(dst)) { + if (!['.md', '.txt'].includes(extname(path))) continue + const rel = relative(dst, path) + if (!EXCLUDE_BASENAMES.includes(basename(path)) && (shouldSkip(rel, options) || copied.has(rel))) continue + await unlink(path); deleted++ + } + return { copied: copied.size, skipped, deleted } +} +export async function postprocessTree(root: string, options: Ownership = {}): Promise<void> { + for (const path of await files(root)) { + if (extname(path) !== '.md' || shouldSkip(relative(root, path), options)) continue + const text = await readText(path) + const next = preserveFences(text, body => normalizeTrailingNewlines(normalizeHeadingsForContext(path, normalizeUserFacingLinks(normalizeEditorialFraming(body))))) + if (next !== text) await writeFile(path, next) + } +} +export async function syncFromSilkDocs(site: string, silkRepo: string): Promise<void> { + const srcDocs = resolve(silkRepo, 'docs'), srcWiki = join(srcDocs, 'wiki') + try { await access(srcDocs) } catch { throw new Error(`Missing source docs at ${srcDocs}`) } + const staged = await mkdtemp(join(tmpdir(), 'oro-silk-import-')) + try { + const dstDocs = join(staged, 'docs'), dstWiki = join(staged, 'wiki') + // Empty collections still need a staging directory for the importer. + await mkdir(dstDocs); await mkdir(dstWiki) + await stageCollection('silk', dstDocs, site) + await stageCollection('silkWiki', dstWiki, site) + const docsStats = await syncTree(srcDocs, dstDocs, { + keepFiles: KEEP_DOCS_FILES, keepPrefixes: [...KEEP_DOCS_PREFIXES, 'wiki/'], + sanitize: (rel, text) => preserveFences(text, body => sanitizeDocsMarkdown(rel, body)), + }) + let hasWiki = true + try { await access(srcWiki) } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + hasWiki = false + } + const wikiStats = hasWiki ? await syncTree(srcWiki, dstWiki, { + keepFiles: KEEP_WIKI_FILES, sanitize: (rel, text) => preserveFences(text, body => sanitizeWikiMarkdown(rel, body)), + }) : { copied: 0, skipped: 0, deleted: 0 } + await postprocessTree(dstDocs, { keepFiles: KEEP_DOCS_FILES, keepPrefixes: KEEP_DOCS_PREFIXES }) + await postprocessTree(dstWiki, { keepFiles: KEEP_WIKI_FILES }) + await importCollection('silk', dstDocs, site) + if (hasWiki) await importCollection('silkWiki', dstWiki, site) + console.log('Synced Silk docs to website:') + console.log(`- Docs copied: ${docsStats.copied} (skipped: ${docsStats.skipped}, deleted: ${docsStats.deleted})`) + console.log(`- Wiki copied: ${wikiStats.copied} (skipped: ${wikiStats.skipped}, deleted: ${wikiStats.deleted})`) + } finally { await rm(staged, { recursive: true, force: true }) } +} +export async function main(args = process.argv.slice(2)): Promise<void> { + const { values } = parseArgs({ args, options: { 'repo-root': { type: 'string' }, 'silk-repo': { type: 'string' }, help: { type: 'boolean', short: 'h' } } }) + if (values.help) { console.log('Sync Silk docs/wiki from the repo (silk/docs) into the website copies (website/silk).\n\nUsage: node silk/tools/sync-from-silk-docs.ts [--repo-root PATH] [--silk-repo PATH]'); return } + const site = fileURLToPath(new URL('../../', import.meta.url)) + await syncFromSilkDocs(site, values['silk-repo'] ?? resolve(values['repo-root'] ?? resolve(site, '..'), 'silk')) +} +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + main().catch(error => { + console.error(error.message) + process.exitCode = error.code?.startsWith('ERR_PARSE_ARGS_') ? 2 : 1 + }) +} diff --git a/slg/tools/audit-site.py b/slg/tools/audit-site.py deleted file mode 100644 index f0f9d0d2..00000000 --- a/slg/tools/audit-site.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import sys -import os -from pathlib import Path - - -SITE_ROOT = Path(__file__).resolve().parents[2] -SLG_ROOT = Path(os.environ.get("ORO_SITE_OUTPUT", str(SITE_ROOT / "public"))) / "slg" -REPO_ROOT = SLG_ROOT.parent - -sys.path.insert(0, str(SITE_ROOT / "tools")) - -from site_audit_common import SiteAuditConfig, run_site_audit - - -def main() -> int: - return run_site_audit( - SiteAuditConfig( - project_root=SLG_ROOT, - docs_source=SLG_ROOT / "docs" / "source", - docs_index=SLG_ROOT / "docs" / "index.json", - missing_index_message="Missing Slg docs/index.json; run build scripts first.", - success_message="OK: slg site audit passed", - ) - ) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/ingestion-fixtures.json b/tests/ingestion-fixtures.json new file mode 100644 index 00000000..183d842c --- /dev/null +++ b/tests/ingestion-fixtures.json @@ -0,0 +1,143 @@ +{ + "cases": [ + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": "# Reference (Current)\n\nStatus: **Partially implemented**. The repository has current supported contexts.\nWorks today: prose\n## Syntax (Selected)\n## Examples (Works today): sample\n## Works today: another\n## What works today (native)\nexamples labeled “Works today”\nexport const region global_arena; const region arena; with arena; from arena; arena[0]; `arena`; `global_arena`\n## Implementation Status\nImplementation status (native): in progress\nInitial Implemented Subset; current compiler subset; active expansion; Partially implemented\nStatus and Future Work\n## Notes and Limitations\n## Current API (native)\n## Design Goals\nKeep design notes.\n## Tests\nRemove test prose.\n### Nested tests\nAlso removed.\n## Arenas\nRemoved arena prose.\n## Surface Syntax\nThe repository, the repository, This repository, this repository.\n`this repository` repo dependency workflow; what works today.\n`STATUS.md` (implementation status) (tracked in `PLAN.md`)\n- End-to-end support snapshot: `STATUS.md`\n- Implemented-subset notes: PLAN.md\n- Relevant fixtures: useful and `STATUS.md`\nStatus: implemented for the current front-end + native backend subset.\nSee `?p=std/runtime`, ?p=docs/language/README and `silk` (1), `std::io` (3).\nSee [docs/std/runtime-event-loop.md](old), docs/std/foo.md, `docs/language/mutability.md`.\nhttps://oro.computer/silk/docs/?p=docs/start https://oro.computer/silk/wiki/?p=wiki/start\n`?p=...` docs/usage/tutorials/README.md ?p=man/my-app.7 docs/std/foo-bar.md\n## Silk Proposal Process (TC39-Inspired)\nRemove proposal.\n## Exported API\nchecker.checkModuleSetWithImports\nCLI output: `silk` CLI and `silk` (1) — Silk Language Compiler (`silk --version`)\n## Current Scope\nMVP initial std wrapper current snapshot.\n", + "expected": "# Reference\n\nThe Silk compiler repository has supported contexts.\nSupported forms: prose\n## Syntax\n## Examples: sample\n## Example: another\n## Supported forms\nexamples labeled “Example”\nexport const region global_region_buf; const region region_buf; with region_buf; from region_buf; region_buf[0]; `region_buf`; `global_region_buf`\n## Notes\navailable\nInitial Supported forms; current implementation; current module surface; Implemented\nStatus and Future Work\n## Considerations\n## Exported API\n## Surface Syntax\nThe Silk compiler repository, the Silk compiler repository, The Silk compiler repository, the Silk compiler repository.\n`this repository` Silk compiler repository’s vendored dependency workflow; current implementation notes.\n[implementation status](?p=compiler/implementation-status)\n- End-to-end support snapshot: [implementation status](?p=compiler/implementation-status)\n- Notes: PLAN.md\n- Relevant fixtures: useful and [implementation status](?p=compiler/implementation-status)\nimplemented for the current front-end + native backend subset.\nSee [runtime](?p=std/runtime), [start](?p=start) and [`silk(1)`](?p=man/silk.1), [`std::io(3)`](?p=std/io).\nSee [runtime](?p=std/runtime), docs/std/foo.md, [mutability](?p=language/mutability).\n[start](?p=start) wiki/wiki/?p=start\ndocs links [start](?p=start) ?p=man/my-app.7 docs/std/foo-bar.md\n## Silk Proposal Process (TC39-Inspired)\nRemove proposal.\n## Exported API\nchecker.checkModuleSetWithImports\nCLI output: `silk` CLI and [`silk(1)`](?p=man/silk.1) — Silk Language Compiler (`silk --version`)\n## Considerations\nbaseline stdlib wrapper current implementation.\n\n## Design goals\nKeep design notes.\n" + }, + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": "```silk\n// Works today: {{ literal }} \nlet n = 1;\n```\n", + "expected": "```silk\n// Works today: {{ literal }} \nlet n = 1;\n```\n" + }, + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": " ~~~~silk\nStatus: **private**.\n~~~\n ~~~~~\n", + "expected": " ~~~~silk\nStatus: **private**.\n~~~\n ~~~~~\n" + }, + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": "````md\n```silk\nthis repository\n```\n````\n", + "expected": "````md\n```silk\nthis repository\n```\n````\n" + }, + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": "```silk\n// Works today: unclosed ", + "expected": "```silk\n// Works today: unclosed " + }, + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": " ```silk\nWorks today: indented\n ```\n", + "expected": " ```silk\nCurrent subset: indented\n ```\n" + }, + { + "rel": "std/sample.md", + "path": "/fixture/std/sample.md", + "kind": "docs", + "input": "```invalid`info\nWorks today: not a fence\n```\n", + "expected": "```invalid`info\nCurrent subset: not a fence\n```\n" + }, + { + "rel": "language/flow-for.md", + "path": "/fixture/language/flow-for.md", + "kind": "docs", + "input": "## Design Goals\nKeep design notes.\n## Current Scope\nMVP initial std wrapper current snapshot.\n", + "expected": "## Model\nKeep design notes.\n## Current Scope\nMVP initial std wrapper current snapshot.\n" + }, + { + "rel": "spec/2026.md", + "path": "/fixture/spec/2026.md", + "kind": "docs", + "input": "## Silk Proposal Process (TC39-Inspired)\nRemove proposal.\n## Exported API\nchecker.checkModuleSetWithImports\nCLI output: `silk` CLI and `silk` (1) — Silk Language Compiler (`silk --version`)\n", + "expected": "## Exported API\nthe module-set import helper\nCLI output: the `silk` CLI and [`silk(1)`](?p=man/silk.1) (`silk --version`)\n" + }, + { + "rel": "compiler/vendored-deps.md", + "path": "/fixture/compiler/vendored-deps.md", + "kind": "docs", + "input": "# Built-In Dependencies\n", + "expected": "# Built-In Dependencies (Legacy Link)\n" + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": "# Reference (Current)\n\nStatus: **Partially implemented**. The repository has current supported contexts.\nWorks today: prose\n## Syntax (Selected)\n## Examples (Works today): sample\n## Works today: another\n## What works today (native)\nexamples labeled “Works today”\nexport const region global_arena; const region arena; with arena; from arena; arena[0]; `arena`; `global_arena`\n## Implementation Status\nImplementation status (native): in progress\nInitial Implemented Subset; current compiler subset; active expansion; Partially implemented\nStatus and Future Work\n## Notes and Limitations\n## Current API (native)\n## Design Goals\nKeep design notes.\n## Tests\nRemove test prose.\n### Nested tests\nAlso removed.\n## Arenas\nRemoved arena prose.\n## Surface Syntax\nThe repository, the repository, This repository, this repository.\n`this repository` repo dependency workflow; what works today.\n`STATUS.md` (implementation status) (tracked in `PLAN.md`)\n- End-to-end support snapshot: `STATUS.md`\n- Implemented-subset notes: PLAN.md\n- Relevant fixtures: useful and `STATUS.md`\nStatus: implemented for the current front-end + native backend subset.\nSee `?p=std/runtime`, ?p=docs/language/README and `silk` (1), `std::io` (3).\nSee [docs/std/runtime-event-loop.md](old), docs/std/foo.md, `docs/language/mutability.md`.\nhttps://oro.computer/silk/docs/?p=docs/start https://oro.computer/silk/wiki/?p=wiki/start\n`?p=...` docs/usage/tutorials/README.md ?p=man/my-app.7 docs/std/foo-bar.md\n## Silk Proposal Process (TC39-Inspired)\nRemove proposal.\n## Exported API\nchecker.checkModuleSetWithImports\nCLI output: `silk` CLI and `silk` (1) — Silk Language Compiler (`silk --version`)\n## Current Scope\nMVP initial std wrapper current snapshot.\n", + "expected": "# Reference\n\nThe repository has supported contexts.\nSupported forms: prose\n## Syntax\n## Examples: sample\n## Example: another\n## Supported forms\nexamples labeled “Example”\nexport const region global_region_buf; const region region_buf; with region_buf; from region_buf; region_buf[0]; `region_buf`; `global_region_buf`\n## Notes\navailable\nInitial Supported forms; current implementation; current module surface; Implemented\nStatus and Future Work\n## Considerations\n## Exported API\n## Design Goals\nKeep design notes.\n## Tests\nRemove test prose.\n### Nested tests\nAlso removed.\n## Arenas\nRemoved arena prose.\n## Surface Syntax\nThe repository, the repository, This repository, this repository.\n`this repository` repo dependency workflow; what works today.\n (implementation status) (tracked in )\n- Fixtures: useful\nImplemented in the reference compiler (front-end + native backend subset).\nSee [runtime](?p=std/runtime), [start](?p=start) and [`silk(1)`](?p=man/silk.1), [`std::io(3)`](?p=std/io).\nSee [runtime](?p=std/runtime), docs/std/foo.md, [mutability](?p=language/mutability).\n[start](?p=start) wiki/wiki/?p=start\ndocs links [start](?p=start) ?p=man/my-app.7 docs/std/foo-bar.md\n## Silk Proposal Process (TC39-Inspired)\nRemove proposal.\n## Exported API\nchecker.checkModuleSetWithImports\nCLI output: `silk` CLI and [`silk(1)`](?p=man/silk.1) — Silk Language Compiler (`silk --version`)\n## Current Scope\nMVP initial std wrapper current snapshot.\n" + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": "```silk\n// Works today: {{ literal }} \nlet n = 1;\n```\n", + "expected": "```silk\n// Works today: {{ literal }} \nlet n = 1;\n```\n" + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": " ~~~~silk\nStatus: **private**.\n~~~\n ~~~~~\n", + "expected": " ~~~~silk\nStatus: **private**.\n~~~\n ~~~~~\n" + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": "````md\n```silk\nthis repository\n```\n````\n", + "expected": "````md\n```silk\nthis repository\n```\n````\n" + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": "```silk\n// Works today: unclosed ", + "expected": "```silk\n// Works today: unclosed " + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": " ```silk\nWorks today: indented\n ```\n", + "expected": " ```silk\nCurrent subset: indented\n ```\n" + }, + { + "rel": "wiki/reference.md", + "path": "/fixture/wiki/reference.md", + "kind": "wiki", + "input": "```invalid`info\nWorks today: not a fence\n```\n", + "expected": "```invalid`info\nCurrent subset: not a fence\n```\n" + } + ], + "curated": [ + { + "input": "# Curated\n\nKeep prose.\n", + "declarations": "declare module 'oro:application' {\n export function ping(): void;\n}\n", + "expected": "# Curated\n\nKeep prose.\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\n\n### Module specifiers\n\n```text\noro:application\n```\n\n### TypeScript declarations\n\nThese declarations are generated from the runtime's published TypeScript surface.\n\n#### `oro:application`\n\n```ts\ndeclare module 'oro:application' {\n export function ping(): void;\n}\n```\n\n<!-- GENERATED: ORO_API_REFERENCE_END -->\n\n## See also\n\n- [JavaScript APIs overview](?p=javascript/overview)\n- [All module specifiers](?p=javascript/all-modules)\n", + "secondExpected": "# Curated\n\nKeep prose.\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\n\n### Module specifiers\n\n```text\noro:application\n```\n\n### TypeScript declarations\n\nThese declarations are generated from the runtime's published TypeScript surface.\n\n#### `oro:application`\n\n```ts\ndeclare module 'oro:application' {\n export function ping(): void;\n}\n```\n\n<!-- GENERATED: ORO_API_REFERENCE_END -->\n## See also\n\n- [JavaScript APIs overview](?p=javascript/overview)\n- [All module specifiers](?p=javascript/all-modules)\n" + }, + { + "input": "# Curated\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\nOld API\n<!-- GENERATED: ORO_API_REFERENCE_END -->\n\nKeep footer.\n", + "declarations": "declare module 'oro:application' {\n export function ping(): void;\n}\n", + "expected": "# Curated\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\n\n### Module specifiers\n\n```text\noro:application\n```\n\n### TypeScript declarations\n\nThese declarations are generated from the runtime's published TypeScript surface.\n\n#### `oro:application`\n\n```ts\ndeclare module 'oro:application' {\n export function ping(): void;\n}\n```\n\n<!-- GENERATED: ORO_API_REFERENCE_END -->\nKeep footer.\n\n## See also\n\n- [JavaScript APIs overview](?p=javascript/overview)\n- [All module specifiers](?p=javascript/all-modules)\n", + "secondExpected": "# Curated\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\n\n### Module specifiers\n\n```text\noro:application\n```\n\n### TypeScript declarations\n\nThese declarations are generated from the runtime's published TypeScript surface.\n\n#### `oro:application`\n\n```ts\ndeclare module 'oro:application' {\n export function ping(): void;\n}\n```\n\n<!-- GENERATED: ORO_API_REFERENCE_END -->\nKeep footer.\n\n## See also\n\n- [JavaScript APIs overview](?p=javascript/overview)\n- [All module specifiers](?p=javascript/all-modules)\n" + }, + { + "input": "# Curated\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\nOld\n<!-- GENERATED: ORO_API_REFERENCE_END -->\n\n## See also\n\nKeep links.\n", + "declarations": "declare module 'oro:application' {\n export function ping(): void;\n}\n", + "expected": "# Curated\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\n\n### Module specifiers\n\n```text\noro:application\n```\n\n### TypeScript declarations\n\nThese declarations are generated from the runtime's published TypeScript surface.\n\n#### `oro:application`\n\n```ts\ndeclare module 'oro:application' {\n export function ping(): void;\n}\n```\n\n<!-- GENERATED: ORO_API_REFERENCE_END -->\n## See also\n\nKeep links.\n", + "secondExpected": "# Curated\n\n## API reference\n\n<!-- GENERATED: ORO_API_REFERENCE_START -->\n\n### Module specifiers\n\n```text\noro:application\n```\n\n### TypeScript declarations\n\nThese declarations are generated from the runtime's published TypeScript surface.\n\n#### `oro:application`\n\n```ts\ndeclare module 'oro:application' {\n export function ping(): void;\n}\n```\n\n<!-- GENERATED: ORO_API_REFERENCE_END -->\n## See also\n\nKeep links.\n" + } + ] +} diff --git a/tests/ingestion-import.test.ts b/tests/ingestion-import.test.ts new file mode 100644 index 00000000..04363463 --- /dev/null +++ b/tests/ingestion-import.test.ts @@ -0,0 +1,85 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { generateReference, REF_START, REF_END } from '../runtime/tools/generate-js-api-reference.ts' +import { syncFromSilkDocs } from '../silk/tools/sync-from-silk-docs.ts' +import { pageText, splitPage } from '../tools/import-public.ts' + +async function temporary(run: (site: string, upstream: string) => Promise<void>): Promise<void> { + const root = await mkdtemp(join(tmpdir(), 'oro-ingestion-import-test-')) + try { + const site = join(root, 'site'), upstream = join(root, 'upstream') + await mkdir(join(site, 'src'), { recursive: true }) + await mkdir(upstream) + await run(site, upstream) + } finally { await rm(root, { recursive: true, force: true }) } +} +async function put(path: string, body: string): Promise<void> { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, body) +} +async function page(site: string, route: string, collection: string, sourcePath: string, body: string): Promise<string> { + const path = join(site, 'src', route, 'page.md') + await put(path, pageText({ layout: collection === 'silk' && sourcePath === 'spec/2026.md' ? 'spec' : 'docs', title: 'Authored title', description: 'Authored description', docsCollection: collection, sourcePath, order: 7, section: 'authored', githubRepo: 'preserved/repository', githubRef: 'authored-ref' }, body)) + return path +} +test('Runtime generator uses the public importer, prunes excluded families, and retains curated metadata', async () => temporary(async (site, upstream) => { + const curated = await page(site, 'runtime/docs/javascript/application', 'runtime', 'javascript/application.md', '# Application\n\nAuthored introduction.\n\n' + REF_START + '\nOld API\n' + REF_END + '\n\nAuthored footer.\n') + const excluded = await page(site, 'runtime/docs/javascript/internal', 'runtime', 'javascript/internal.md', '# Internal\n') + const unrelated = await page(site, 'runtime/docs/guides/custom', 'runtime', 'guides/custom.md', '# Guide\n\nWorks today: authored {{ literal }}.\n') + const untouched = await readFile(unrelated, 'utf8') + await put(join(upstream, 'api/index.d.ts'), "declare module 'oro:application' {\n export function ping(): void;\n}\ndeclare module 'oro:assert' {\n export function ok(value: unknown): void;\n}\ndeclare module 'oro:internal' {\n export const privateAPI: true;\n}\n") + await generateReference(site, upstream) + const { metadata, body } = splitPage(await readFile(curated, 'utf8')) + assert.ok(body.includes('Authored introduction.')) + assert.ok(body.includes('Authored footer.')) + assert.ok(body.includes('ping()')) + assert.ok(!body.includes('Old API')) + assert.equal(metadata.layout, 'docs') + assert.equal(metadata.order, 7) + assert.equal(metadata.githubRef, 'authored-ref') + assert.equal(metadata.section, 'authored') + assert.equal(await readFile(unrelated, 'utf8'), untouched) + await assert.rejects(stat(excluded), { code: 'ENOENT' }) + const generated = join(site, 'src/runtime/docs/javascript/assert/page.md') + const first = await readFile(generated, 'utf8') + assert.equal(splitPage(first).metadata.layout, 'docs') + await generateReference(site, upstream) + assert.equal(await readFile(generated, 'utf8'), first) +})) +test('Silk imports prune upstream-owned pages but retain authored pages and an absent upstream wiki', async () => temporary(async (site, upstream) => { + const landing = await page(site, 'silk/docs/start', 'silk', 'start.md', '# Silk\n\nWorks today: authored.\n') + const guide = await page(site, 'silk/docs/guides/custom', 'silk', 'guides/custom.md', '# Guide\n\nAuthored guide.\n') + const wiki = await page(site, 'silk/wiki/reference', 'silkWiki', 'reference.md', '# Wiki\n\nWorks today: preserve without upstream wiki.\n') + const removed = await page(site, 'silk/docs/language/removed', 'silk', 'language/removed.md', '# Removed\n') + const originals = await Promise.all([landing, guide, wiki].map(path => readFile(path, 'utf8'))) + await put(join(upstream, 'docs/start.md'), '# Upstream start\n') + await put(join(upstream, 'docs/guides/custom.md'), '# Upstream guide\n') + await put(join(upstream, 'docs/spec/2026.md'), '# Silk specification\n') + await put(join(upstream, 'docs/language/new.md'), '# New\n\nWorks today: prose\n\n```silk\n// Works today: {{ literal }} \n```\n') + await syncFromSilkDocs(site, upstream) + assert.deepEqual(await Promise.all([landing, guide, wiki].map(path => readFile(path, 'utf8'))), originals) + await assert.rejects(stat(removed), { code: 'ENOENT' }) + const added = join(site, 'src/silk/docs/language/new/page.md') + const first = await readFile(added, 'utf8') + assert.equal(splitPage(first).metadata.layout, 'docs') + assert.equal(splitPage(await readFile(join(site, 'src/silk/spec/2026/page.md'), 'utf8')).metadata.layout, 'spec') + assert.ok(first.includes('// Works today: {{ literal }} \n')) + await syncFromSilkDocs(site, upstream) + assert.equal(await readFile(added, 'utf8'), first) + await put(join(upstream, 'docs/wiki/fresh.md'), '# Fresh wiki\n') + await syncFromSilkDocs(site, upstream) + await assert.rejects(stat(wiki), { code: 'ENOENT' }) + const fresh = splitPage(await readFile(join(site, 'src/silk/wiki/fresh/page.md'), 'utf8')) + assert.equal(fresh.metadata.layout, 'docs') + assert.ok(fresh.body.includes('# Fresh wiki')) +})) +test('Runtime validation failures do not partially import staged changes', async () => temporary(async (site, upstream) => { + const curated = await page(site, 'runtime/docs/javascript/application', 'runtime', 'javascript/application.md', '# Authored\n') + const original = await readFile(curated, 'utf8') + await put(join(upstream, 'api/index.d.ts'), "declare module 'oro:application' {\n export function ping(): void;\n}\ndeclare module 'oro:unknown' {\n}\n") + await assert.rejects(generateReference(site, upstream), /Missing generated examples/) + assert.equal(await readFile(curated, 'utf8'), original) +})) diff --git a/tests/ingestion.test.ts b/tests/ingestion.test.ts index 89f53629..7d954123 100644 --- a/tests/ingestion.test.ts +++ b/tests/ingestion.test.ts @@ -1,5 +1,119 @@ import { test } from 'node:test' -import { execFileSync } from 'node:child_process' -test('manual upstream generators preserve curated content and ingestion ownership', () => { - execFileSync('python3', ['tests/test_ingestion.py'], { stdio: 'inherit' }) +import assert from 'node:assert/strict' +import { readFile, writeFile, mkdir, mkdtemp, rm, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, relative } from 'node:path' +import { execFileSync, spawnSync } from 'node:child_process' +import { files, preserveFences, splitLines } from '../tools/source-pages.ts' +import { KEEP_DOCS_FILES, KEEP_DOCS_PREFIXES, syncTree, postprocessTree, sanitizeDocsMarkdown, sanitizeWikiMarkdown, normalizeEditorialFraming, normalizeUserFacingLinks, normalizeHeadingsForContext, normalizeTrailingNewlines, main as silkMain } from '../silk/tools/sync-from-silk-docs.ts' +import { parseIndexDTS, renderGeneratedPage, updateCuratedPage } from '../runtime/tools/generate-js-api-reference.ts' +import { DESCRIPTION_BY_FAMILY, EXAMPLES_BY_FAMILY, GUIDE_REFS_BY_FAMILY, DEFAULT_SEE_ALSO } from '../runtime/tools/js-api-reference-content.ts' + +const fixture = JSON.parse(await readFile(new URL('./ingestion-fixtures.json', import.meta.url), 'utf8')) as { + cases: { rel: string; path: string; kind: string; input: string; expected: string }[] + curated: { input: string; declarations: string; expected: string; secondExpected: string }[] +} +async function temporary(run: (root: string) => Promise<void>): Promise<void> { + const root = await mkdtemp(join(tmpdir(), 'oro-ingestion-test-')) + try { await run(root) } finally { await rm(root, { recursive: true, force: true }) } +} +test('configured runtime families render descriptions, examples, links, and ordered declarations', () => { + const families = Object.keys(EXAMPLES_BY_FAMILY) + assert.ok(families.length > 0) + for (const family of families) { + const specs = [family + '/z', family, family + '/index'] + const blocks = parseIndexDTS(specs.map(spec => `declare module '${spec}' {\n export function ping(): void;\n}\n`).join('\n')) + const rendered = renderGeneratedPage('unused', family, specs, blocks) + assert.ok(rendered.startsWith(`# \`${family}\`\n`), family) + assert.ok(rendered.includes(DESCRIPTION_BY_FAMILY[family].trim()), family) + assert.ok(rendered.includes(`## Examples\n\n${EXAMPLES_BY_FAMILY[family].trimEnd()}`), family) + assert.ok(rendered.includes(`\n${family}\n${family}/index\n${family}/z\n`), family) + const ordered = [family, family + '/index', family + '/z'] + for (const spec of ordered) assert.ok(rendered.includes(blocks[spec].block.trimEnd()), spec) + assert.deepEqual([...rendered.matchAll(/^#### `([^`]+)`$/gm)].map(match => match[1]), ordered) + const guides = GUIDE_REFS_BY_FAMILY[family] ?? [] + assert.equal(rendered.includes('## Related guides'), guides.length > 0, family) + for (const [label, id] of [...guides, ...DEFAULT_SEE_ALSO]) assert.ok(rendered.includes(`- [${label}](?p=${id})`), family) + assert.ok(rendered.endsWith('\n') && !rendered.endsWith('\n\n'), family) + } +}) +test('Silk editorial transformations preserve context and fenced content', () => { + for (const [index, entry] of fixture.cases.entries()) { + const sanitize = entry.kind === 'wiki' ? sanitizeWikiMarkdown : sanitizeDocsMarkdown + let actual = preserveFences(entry.input, body => sanitize(entry.rel, body)) + actual = preserveFences(actual, body => normalizeTrailingNewlines(normalizeHeadingsForContext(entry.path, normalizeUserFacingLinks(normalizeEditorialFraming(body))))) + assert.equal(actual, entry.expected, `fixture ${index}: ${entry.rel}`) + } +}) +test('curated reference markers preserve surrounding prose and are idempotent', async () => temporary(async root => { + const path = join(root, 'application.md') + for (const entry of fixture.curated) { + await writeFile(path, entry.input) + const blocks = parseIndexDTS(entry.declarations) + assert.equal(await updateCuratedPage(path, 'oro:application', Object.keys(blocks), blocks), true) + assert.equal(await readFile(path, 'utf8'), entry.expected) + // The original normalizes the gap before a newly appended See also on its second pass. + await updateCuratedPage(path, 'oro:application', Object.keys(blocks), blocks) + assert.equal(await readFile(path, 'utf8'), entry.secondExpected) + const before = (await stat(path)).mtimeMs + assert.equal(await updateCuratedPage(path, 'oro:application', Object.keys(blocks), blocks), false) + assert.equal((await stat(path)).mtimeMs, before) + } +})) +test('Silk ownership, exclusions, pruning, raw txt copying, and postprocessing', async () => temporary(async root => { + const src = join(root, 'upstream'), dst = join(root, 'staged') + for (const base of [src, dst]) for (const name of ['guides', 'language', 'wiki']) await mkdir(join(base, name), { recursive: true }) + const authored = '# Custom guide\n\nWorks today: a deliberately authored phrase.\n' + for (const name of ['guides/custom.md', 'start.md']) { + await writeFile(join(src, name), 'upstream') + await writeFile(join(dst, name), authored) + } + await writeFile(join(src, 'language/new.md'), '# New\n\nWorks today: prose\n\n```silk\n// Works today: {{ literal }} \n```\n') + await writeFile(join(src, 'language/raw.txt'), Buffer.from('raw\r\nbytes\u0000')) + await writeFile(join(src, 'wiki/reference.md'), 'wiki') + await writeFile(join(src, 'STATUS.md'), 'private tracker') + await writeFile(join(src, 'ignored.bin'), 'ignored') + await writeFile(join(dst, 'language/removed.md'), 'removed') + await writeFile(join(dst, 'STATUS.md'), 'stale tracker') + await writeFile(join(dst, 'asset.svg'), 'not owned') + await writeFile(join(dst, 'guides/PLAN.md'), 'excluded even under an owned prefix') + const ownership = { keepFiles: KEEP_DOCS_FILES, keepPrefixes: [...KEEP_DOCS_PREFIXES, 'wiki/'] } + assert.deepEqual(await syncTree(src, dst, { ...ownership, sanitize: (rel, text) => preserveFences(text, body => sanitizeDocsMarkdown(rel, body)) }), { copied: 2, skipped: 4, deleted: 3 }) + await postprocessTree(dst, ownership) + for (const name of ['guides/custom.md', 'start.md']) assert.equal(await readFile(join(dst, name), 'utf8'), authored) + for (const name of ['language/removed.md', 'STATUS.md', 'guides/PLAN.md', 'wiki/reference.md', 'ignored.bin']) await assert.rejects(stat(join(dst, name)), { code: 'ENOENT' }) + assert.deepEqual(await readFile(join(dst, 'language/raw.txt')), await readFile(join(src, 'language/raw.txt'))) + assert.equal(await readFile(join(dst, 'language/new.md'), 'utf8'), '# New\n\nSupported forms: prose\n\n```silk\n// Works today: {{ literal }} \n```\n') + assert.equal(await readFile(join(dst, 'asset.svg'), 'utf8'), 'not owned') + assert.deepEqual((await files(dst)).map(path => relative(dst, path)).sort(), [ + 'asset.svg', 'guides/custom.md', 'language/new.md', 'language/raw.txt', 'start.md', + ]) +})) +test('fence placeholders cannot collide with authored text; mismatched and unclosed fences stay exact', () => { + const fence = '````silk\nWorks today: {{ value }} \n```\n~~~~\n`````\n' + const text = 'OROFENCE0END\n\nWorks today: prose\n\n' + fence + assert.equal(preserveFences(text, body => body.replaceAll('Works today', 'Supported')), text.replace('Works today: prose', 'Supported: prose')) + const unclosed = ' ~~~~silk\nWorks today: literal ' + assert.equal(preserveFences(unclosed, body => body.trimEnd() + '\n'), unclosed) +}) +test('declaration parsing retains duplicate-last behavior, nesting, whitespace, and errors', () => { + assert.throws(() => parseIndexDTS('not a declaration'), /No `declare module/) + assert.throws(() => renderGeneratedPage('', 'oro:unknown', [], {}), /Missing generated examples/) + const text = "declare module 'oro:x' {\r\n export interface X {\r\n nested: string; \r\n }\r\n}\r\ndeclare module \"oro:x\" {\r\n export const last: true; \r\n}\r\n" + assert.equal(parseIndexDTS(text)['oro:x'].block, 'declare module "oro:x" {\n export const last: true;\n}\n') + assert.deepEqual(splitLines('a\n\n'), ['a', '']) + assert.deepEqual(splitLines('a\r\n'), ['a']) + assert.deepEqual(splitLines('a\u0085b\u2028c\v'), ['a', 'b', 'c']) +}) +test('Silk CLI resolves repo-root and gives silk-repo precedence', async () => temporary(async root => { + await assert.rejects(silkMain(['--repo-root', root]), { message: `Missing source docs at ${join(root, 'silk/docs')}` }) + await assert.rejects(silkMain(['--repo-root', root, '--silk-repo', join(root, 'explicit')]), { message: `Missing source docs at ${join(root, 'explicit/docs')}` }) +})) +test('manual CLI entry points expose preserved flags without running imports', () => { + for (const [path, flag] of [['runtime/tools/generate-js-api-reference.ts', '--runtime-repo'], ['silk/tools/sync-from-silk-docs.ts', '--silk-repo']]) { + const help = execFileSync(process.execPath, [path, '--help'], { encoding: 'utf8' }) + assert.ok(help.includes(flag)) + assert.equal(spawnSync(process.execPath, [path, '--unknown']).status, 2) + assert.equal(spawnSync(process.execPath, [path, flag]).status, 2) + } }) diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py deleted file mode 100644 index f1148ac9..00000000 --- a/tests/test_ingestion.py +++ /dev/null @@ -1,80 +0,0 @@ -import importlib.util -import sys -import tempfile -import unittest -from pathlib import Path - -SITE = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(SITE / 'runtime' / 'tools')) - - -def module(name, path): - spec = importlib.util.spec_from_file_location(name, SITE / path) - result = importlib.util.module_from_spec(spec) - sys.modules[name] = result - spec.loader.exec_module(result) - return result - - -silk = module('silk_sync', 'silk/tools/sync-from-silk-docs.py') -runtime = module('runtime_api', 'runtime/tools/generate-js-api-reference.py') - - -class IngestionTests(unittest.TestCase): - def test_silk_preserves_website_owned_pages_and_prunes_removed_imports(self): - with tempfile.TemporaryDirectory() as folder: - src, dst = Path(folder) / 'upstream', Path(folder) / 'staged' - for base in (src, dst): - (base / 'guides').mkdir(parents=True) - (base / 'language').mkdir() - (src / 'guides' / 'custom.md').write_text('upstream guide') - (dst / 'guides' / 'custom.md').write_text('website guide') - (src / 'start.md').write_text('upstream start') - (dst / 'start.md').write_text('website start') - (src / 'language' / 'new.md').write_text('# New\n') - (dst / 'language' / 'removed.md').write_text('# Removed\n') - (src / 'STATUS.md').write_text('private tracker') - silk.sync_tree(src, dst, keep_files=silk.KEEP_DOCS_FILES, - keep_prefixes=silk.KEEP_DOCS_PREFIXES) - self.assertEqual((dst / 'guides/custom.md').read_text(), 'website guide') - self.assertEqual((dst / 'start.md').read_text(), 'website start') - self.assertTrue((dst / 'language/new.md').exists()) - self.assertFalse((dst / 'language/removed.md').exists()) - self.assertFalse((dst / 'STATUS.md').exists()) - - def test_postprocessing_leaves_website_owned_copy_untouched(self): - with tempfile.TemporaryDirectory() as folder: - root = Path(folder) - (root / 'guides').mkdir() - page = root / 'guides/custom.md' - text = '# Custom guide\n\nWorks today: a deliberately authored phrase.\n' - page.write_text(text) - silk.postprocess_tree(root, keep_files=silk.KEEP_DOCS_FILES, - keep_prefixes=silk.KEEP_DOCS_PREFIXES) - self.assertEqual(page.read_text(), text) - - def test_runtime_preserves_curated_prose_around_generated_markers(self): - with tempfile.TemporaryDirectory() as folder: - path = Path(folder) / 'application.md' - path.write_text('# Curated application\n\nKeep this example.\n\n' + - runtime.REF_START + '\nOld API\n' + runtime.REF_END + '\n\nKeep this footer.\n') - declarations = "declare module 'oro:application' {\n export function ping(): void;\n}\n" - blocks = runtime.parse_index_d_ts(declarations) - runtime.update_curated_page(path, 'oro:application', ['oro:application'], blocks) - result = path.read_text() - self.assertIn('Keep this example.', result) - self.assertIn('Keep this footer.', result) - self.assertIn('ping()', result) - self.assertNotIn('Old API', result) - self.assertEqual(result.count(runtime.REF_START), 1) - self.assertEqual(result.count(runtime.REF_END), 1) - - def test_silk_editorial_pass_preserves_code_comments(self): - example = '```silk\n// Works today: {{ literal }}\nlet n = 1;\n```' - result = silk.source_pages.preserve_fences('Works today: prose\n\n' + example, - silk.normalize_shared_markdown) - self.assertIn(example, result) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/tooling/content-audit.test.ts b/tests/tooling/content-audit.test.ts new file mode 100644 index 00000000..45fb64fc --- /dev/null +++ b/tests/tooling/content-audit.test.ts @@ -0,0 +1,160 @@ +import assert from 'node:assert/strict' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' + +import { test } from 'node:test' +import { auditContext, runContentAudit } from '../../tools/audit-content.ts' +import { buildSourceInventory, runSiteAudit } from '../../tools/audits/common.ts' +import type { AuditContext, Reporter } from '../../tools/audits/common.ts' +import { runRuntimeAudit } from '../../tools/audits/runtime.ts' +import { runSilkAudit, runStdlibAudit } from '../../tools/audits/silk.ts' + +const siteRoot = resolve(import.meta.dirname, '../..') +const cases = ['missing', 'clean', 'common', 'silk', 'stdlib', 'runtime', 'cap'] as const +for (const name of cases) test(`Content audit regression: ${name}`, () => { + const root = mkdtempSync(join(siteRoot, '.audit-fixture-')) + const context: AuditContext = { siteRoot: root, outputRoot: join(root, 'output'), runtimeRepo: join(root, 'output/upstream') } + const put = (path: string, text: string): void => { + mkdirSync(dirname(join(context.outputRoot, path)), { recursive: true }) + writeFileSync(join(context.outputRoot, path), text) + } + const page = (collection: string, sourcePath: string, route = `${collection}/${sourcePath}`): void => { + const path = join(root, 'src', route, 'page.md') + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, `---\ndocsCollection: ${collection}\nsourcePath: ${sourcePath}\n---\n# Source\n`) + } + try { + for (const product of ['runtime', 'silk', 'sage', 'slg', 'virtnosis']) page(product, 'start.md') + if (name !== 'missing') { + for (const product of ['runtime', 'silk', 'sage', 'slg', 'virtnosis']) { + put(`${product}/docs/source/start.md`, '# Start\n') + } + for (const path of ['index.html', 'runtime/index.html', 'runtime/docs/index.html']) put(path, '') + } + if (name === 'common') { + page('sage', 'lost.md') + put('sage/docs/source/start.md', '# Heading ?p=heading\n```\n?p=fenced\n## Status\n~~~\nRaw ?p=missing and `open` (3)\n[ok](?p=start) [bad](wiki/?p=nope)\n[missing](docs/absent.md) [escape](../ignored.md)\n[absolute](/absent.txt) [remote](https://example.org/no.md)\n## Status (details)\n## API (initial surface)\n## Thing (Planned soon)\nStatus: **yes**\nInitial implementation\n## Current API\n## Future Work\n## Current APİ\n\u001c## Status\n\ufeff## Status\néinitial implementationé\n'); + } + if (name === 'silk') { + page('silkWiki', 'wiki.md') + page('silk', 'spec/2026.md', 'silk/spec/2026') + put('silk/docs/source/start.md', '[wiki](wiki/?p=lost) [spec](spec/missing.md)\n```\nconst region arena\nWorks today\n```\nRaw ?p=oops `foo` (1)\n## Status\n') + put('silk/wiki/source/wiki.md', 'from arena\nSyntax (Selected)\n## Current API\n[docs](docs/missing.txt)\n') + put('silk/docs/source/spec/2026.md', 'checker.foo src/example c-tests/ tests/silk/') + } + if (name === 'stdlib') put('silk/docs/source/std/a.md', 'active expansion; current snapshot; Partially implemented; initial std wrapper; expansion path; MVP\n## Examples\n## Description\n## API\n## Design Goals\n## Scope (Current)\n## Initial Design MVP\n```\n## Current Scope\n```\n## Examples\n') + if (name === 'runtime') { + put('runtime/docs/source/javascript/module-index.md', 'console.log(Object.keys(api))') + put('runtime/docs/source/javascript/foo.md', '## Example\n') + put('runtime/docs/source/javascript/node.md', '## Examples\n') + put('runtime/docs/source/javascript/all-modules.md', 'oro:node alone; oro:internal/*; oro:fs') + put('upstream/api/index.d.ts', 'declare module "oro:fs" {}\ndeclare module "oro:fs/promises" {}\ndeclare module "oro:new" {}\ndeclare module "oro:node" {}\n') + put('upstream/api/CLI.md', '## oroc\n## oroc update-init\n## oroc unknown\n') + put('index.html', 'oro-computer/oro-runtime oro-computer/legacy-runtime') + } + if (name === 'cap') put('sage/docs/source/start.md', Array.from({ length: 205 }, (_, i) => `[x](?p=missing${i})`).join('\n')) + const runners: Record<string, (ctx: AuditContext, reporter: Reporter) => number> = { + missing: runContentAudit, clean: runContentAudit, common: (ctx, reporter) => runSiteAudit(ctx, 'sage', reporter), + silk: runSilkAudit, stdlib: runStdlibAudit, runtime: runRuntimeAudit, cap: (ctx, reporter) => runSiteAudit(ctx, 'sage', reporter), + } + const stdout: string[] = [], stderr: string[] = [] + const status = runners[name]!(context, { log: message => stdout.push(message), error: message => stderr.push(message) }) + assert.equal(status, name === 'clean' ? 0 : 1) + if (name === 'clean') { + assert.deepEqual(stdout, [ + 'OK: runtime site audit passed', 'OK: runtime docs audit passed', 'OK: silk site audit passed', + 'Stdlib doc audit passed.', ...['sage', 'slg', 'virtnosis'].map(product => `OK: ${product} site audit passed`), + ]) + assert.deepEqual(stderr, []) + } else if (name === 'cap') { + assert.deepEqual(stdout, []) + assert.equal(stderr.length, 202) + assert.deepEqual(stderr.slice(0, 200).map(message => /Broken \?p= link: missing(\d+)$/.exec(message)?.[1]), + Array.from({ length: 200 }, (_, i) => String(i))) + assert.deepEqual(stderr.slice(200), ['... and 5 more', 'FAIL: 205 issues']) + } else { + // Count every detection category without snapshotting wording or traversal order. + const expected: Record<string, [RegExp, number][]> = { + missing: [[/runtime\/docs\/source\/start.md: Missing raw output/, 1]], + common: [ + [/lost.md: Missing raw output/, 1], [/Broken \?p= link: (heading|fenced|missing|nope)$/, 4], + [/Missing link target: (docs\/absent.md|\/absent.txt) ->/, 2], + [/Line 6: raw \?p= reference/, 1], [/Line 6: raw manpage reference/, 1], + [/Line (10|11|12|13|14|15|16|17|19): avoid status-style/, 9], + ], + silk: [ + [/Broken \?p= link: (wiki\/lost|docs\/oops)$/, 2], + [/Missing link target: (spec\/missing.md|docs\/missing.txt) -> silk\/docs\/source\//, 2], + [/Found arena identifier/, 2], [/Found deprecated/, 2], + [/Line 6: raw \?p= reference/, 1], [/Line 6: raw manpage reference/, 1], + [/Line (3|7): avoid status-style/, 2], [/spec\/2026.md: Spec contains repo-internal/, 1], + ], + stdlib: [ + [/Status\/body uses banned transitional wording:/, 6], + [/Banned heading: (API|Design Goals)$/, 2], + [/Banned heading pattern: Initial Design MVP$/, 2], [/Banned heading pattern: Current Scope$/, 1], + ], + runtime: [ + [/module-index.md: remove this page/, 1], [/module-index.md: remove generic Object.keys/, 1], + [/module-index.md: missing an Examples section/, 1], + [/missing docs page for published module family oro:(fs|new)\.$/, 2], + [/excluded private module family oro:node should not have a public docs page/, 1], + [/excluded private module family oro:internal should not appear in the public module listing/, 1], + [/missing published module specifier oro:(fs\/promises|new)\.$/, 2], + [/cli\/(oroc|update\/init).md: missing website docs page for upstream CLI section/, 2], + [/index.html: stale runtime repository reference/, 3], + ], + } + const findings = name === 'stdlib' ? stdout : stderr.slice(0, -1) + for (const [pattern, count] of expected[name]!) { + assert.equal(findings.filter(message => pattern.test(message)).length, count, String(pattern)) + } + assert.equal(findings.length, expected[name]!.reduce((sum, [, count]) => sum + count, 0)) + if (name === 'stdlib') assert.deepEqual(stderr, []) + else { + assert.deepEqual(stdout, name === 'runtime' ? ['OK: runtime site audit passed'] : []) + assert.equal(stderr.at(-1), `FAIL: ${findings.length} ${name === 'runtime' ? 'runtime docs issues' : 'issues'}`) + } + } + } finally { rmSync(root, { recursive: true, force: true }) } +}) +test('source inventory includes dedicated spec routes and txt outputs without public indexes', () => { + const root = mkdtempSync(join(siteRoot, '.audit-fixture-')) + const context: AuditContext = { siteRoot: root, outputRoot: join(root, 'custom-output') } + const put = (path: string, text: string): void => { + mkdirSync(dirname(join(root, path)), { recursive: true }) + writeFileSync(join(root, path), text) + } + const messages: string[] = [] + const reporter: Reporter = { log: () => {}, error: message => messages.push(message) } + try { + put('src/silk/spec/2026/page.md', '---\ndocsCollection: silk\nsourcePath: spec/2026.md\n---\n') + put('src/silk/docs/reference/page.md', '---\ndocsCollection: silk\nsourcePath: reference.txt\n---\n') + put('src/silk/wiki/start/page.md', '---\ndocsCollection: silkWiki\nsourcePath: start.md\n---\n') + put('src/unrelated/page.md', '---\ntitle: Not documentation\n---\n') + assert.deepEqual(buildSourceInventory(root).silk.map(item => item.file).sort(), ['reference.txt', 'spec/2026.md']) + put('custom-output/silk/docs/source/spec/2026.md', '[text](?p=reference) [spec](?p=spec/2026) [wiki](wiki/?p=start)\n') + put('custom-output/silk/wiki/source/start.md', '[spec](docs/?p=spec/2026)\n') + assert.equal(runSilkAudit(context, reporter), 1) + assert.deepEqual(messages, ['silk/docs/source/reference.txt: Missing raw output for source page; run build scripts first.', 'FAIL: 1 issues']) + messages.length = 0 + put('custom-output/silk/docs/source/reference.txt', 'Reference\n') + assert.equal(runSilkAudit(context, reporter), 0) + // A later run on the same context must observe changed source metadata. + put('src/silk/docs/new/page.md', '---\ndocsCollection: silk\nsourcePath: new.txt\n---\n') + assert.equal(runSilkAudit(context, reporter), 1) + assert.deepEqual(messages, ['silk/docs/source/new.txt: Missing raw output for source page; run build scripts first.', 'FAIL: 1 issues']) + messages.length = 0 + put('custom-output/silk/docs/source/new.txt', 'New\n') + rmSync(join(root, 'custom-output/silk/docs/source/spec/2026.md')) + assert.equal(runSilkAudit(context, reporter), 1) + assert.deepEqual(messages, ['silk/docs/source/spec/2026.md: Missing raw output for source page; run build scripts first.', 'FAIL: 1 issues']) + } finally { rmSync(root, { recursive: true, force: true }) } +}) + +test('audit environment paths', () => { + assert.equal(auditContext({}).outputRoot, join(siteRoot, 'public')) + assert.equal(auditContext({}).runtimeRepo, undefined) + assert.equal(auditContext({ ORO_SITE_OUTPUT: 'other', ORO_RUNTIME_REPO: 'runtime-repo' }).outputRoot, resolve('other')) + assert.equal(auditContext({ ORO_RUNTIME_REPO: 'runtime-repo' }).runtimeRepo, resolve('runtime-repo')) +}) diff --git a/tools/audit-content.ts b/tools/audit-content.ts new file mode 100644 index 00000000..8844d410 --- /dev/null +++ b/tools/audit-content.ts @@ -0,0 +1,31 @@ +import { resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { runSiteAudit, withSourceInventory } from './audits/common.ts' +import type { AuditContext, Reporter } from './audits/common.ts' +import { runRuntimeAudit } from './audits/runtime.ts' +import { runSilkAudit, runStdlibAudit } from './audits/silk.ts' + +export function auditContext(env: NodeJS.ProcessEnv = process.env): AuditContext { + const siteRoot = fileURLToPath(new URL('../', import.meta.url)).replace(/\/$/, '') + return { + siteRoot, + outputRoot: resolve(env.ORO_SITE_OUTPUT ?? resolve(siteRoot, 'public')), + runtimeRepo: env.ORO_RUNTIME_REPO === undefined ? undefined : resolve(env.ORO_RUNTIME_REPO), + } +} +export function runContentAudit(context = auditContext(), reporter: Reporter = console): number { + context = withSourceInventory(context) + // Match the former package script's order and && short-circuit behavior. + const audits = [runRuntimeAudit, runSilkAudit, runStdlibAudit, + ...(['sage', 'slg', 'virtnosis'] as const).map(collection => + (ctx: AuditContext, output: Reporter) => runSiteAudit(ctx, collection, output)), + ] + for (const audit of audits) { + const rc = audit(context, reporter) + if (rc !== 0) return rc + } + return 0 +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exitCode = runContentAudit() +} diff --git a/tools/audits/common.ts b/tools/audits/common.ts new file mode 100644 index 00000000..9f6e1df7 --- /dev/null +++ b/tools/audits/common.ts @@ -0,0 +1,194 @@ +import { existsSync, readFileSync, readdirSync, realpathSync } from 'node:fs' +import { dirname, isAbsolute, relative, resolve } from 'node:path' +import { load } from 'js-yaml' +import { collections, type Collection } from '#lib/collections.ts' + +export interface Issue { path: string; message: string } +export interface AuditContext { siteRoot: string; outputRoot: string; runtimeRepo?: string; inventory?: SourceInventory } +interface SourceItem { id: string; file: string } +export type SourceInventory = Record<Collection, SourceItem[]> +export interface CollectionConfig { source: string; items: SourceItem[]; kind: string } + +export function buildSourceInventory(siteRoot: string): SourceInventory { + const inventory = Object.fromEntries(Object.keys(collections).map(key => [key, []])) as unknown as SourceInventory + for (const path of markdownFiles(pathJoin(siteRoot, 'src')).sort()) { + if (!path.endsWith('/page.md')) continue + const frontmatter = /^---\n([\s\S]*?)\n---(?:\n|$)/.exec(read(path)) + if (!frontmatter) continue + const data = load(frontmatter[1]!) as Record<string, unknown> | undefined + if (!data || typeof data.docsCollection !== 'string' || !Object.hasOwn(collections, data.docsCollection)) continue + if (typeof data.sourcePath !== 'string' || !/\.(md|txt)$/.test(data.sourcePath) || normalizeRelPath(data.sourcePath) !== data.sourcePath) { + throw new Error(`${path}: invalid or missing sourcePath`) + } + inventory[data.docsCollection as Collection].push({ id: data.sourcePath.replace(/\.(md|txt)$/, ''), file: data.sourcePath }) + } + return inventory +} + +// Scope reuse to this invocation, never a process-global cache of source metadata. +export function withSourceInventory(context: AuditContext): AuditContext { + return context.inventory ? context : { ...context, inventory: buildSourceInventory(context.siteRoot) } +} +export type Reporter = Pick<Console, 'log' | 'error'> + +// Translate the Python regex constructs used by these audits, without changing policy. +// In particular: Unicode word/space classes, dotted/dotless I under IGNORECASE, +// and Python's newline-only dot and multiline anchors. +export function pattern(source: string, flags = ''): RegExp { + const word = '[\\p{L}\\p{N}_]' + let translated = '' + let inClass = false + for (let index = 0; index < source.length; index++) { + const char = source[index]! + if (char === '\\') { + const next = source[++index]! + translated += next === 's' ? '[\\p{White_Space}\\x1c-\\x1f]' + : next === 'b' ? `(?:(?<!${word})(?=${word})|(?<=${word})(?!${word}))` : `\\${next}` + } else if (char === '[') { + inClass = true + translated += char + if (flags.includes('i') && /[iI]|a-z|A-Z/.test(source.slice(index + 1, source.indexOf(']', index)))) translated += 'İı' + } else if (char === ']') { inClass = false; translated += char } + else if (!inClass && char === '.') translated += '[^\\n]' + else if (!inClass && char === '^') translated += flags.includes('m') ? '(?:^|(?<=\\n))' : '^' + else if (!inClass && char === '$') translated += flags.includes('m') ? '(?=\\n|$)' : '(?=\\n?$)' + else if (!inClass && flags.includes('i') && /[iI]/.test(char)) translated += '[iIİı]' + else translated += char + } + return new RegExp(translated, flags.replace('m', '') + 'u') +} +export const trim = (text: string): string => text.replace(pattern('^\\s+|\\s+$', 'g'), '') +export const lines = (text: string): string[] => { + const result = text.split(/\r\n|[\n\r\v\f\x1c-\x1e\x85\u2028\u2029]/u) + if (result.at(-1) === '') result.pop() + return result +} +export const read = (path: string): string => readFileSync(path, 'utf8').replace(/\r\n?/g, '\n') +// pathlib joins normalize dots and repeated separators, but not parent segments. +export function pathJoin(...parts: string[]): string { + let value = '' + for (const part of parts) value = isAbsolute(part) || !value ? part : `${value}/${part}` + const absolute = value.startsWith('/') + return (absolute ? '/' : '') + value.split('/').filter(p => p && p !== '.').join('/') +} +export function collectionConfig(context: AuditContext, collection: Collection): CollectionConfig { + const root = pathJoin(context.outputRoot, collections[collection].base.slice(1)) + return { source: pathJoin(root, 'source'), items: (context.inventory ?? buildSourceInventory(context.siteRoot))[collection], kind: collection === 'silkWiki' ? 'wiki' : 'docs' } +} +export function markdownFiles(root: string, recursive = true): string[] { + if (!existsSync(root)) return [] + const entries = readdirSync(root, { withFileTypes: true }) + const files = entries.filter(e => e.name.endsWith('.md')).map(e => pathJoin(root, e.name)) + if (recursive) for (const entry of entries) { + if (entry.isDirectory()) files.push(...markdownFiles(pathJoin(root, entry.name))) + } + return files +} +export const collectionIds = (config: CollectionConfig): Set<string> => new Set(config.items.map(item => item.id)) +export function checkRawOutputs(config: CollectionConfig): Issue[] { + return config.items.filter(item => !existsSync(pathJoin(config.source, item.file))) + .map(item => ({ path: pathJoin(config.source, item.file), message: 'Missing raw output for source page; run build scripts first.' })) +} +export function normalizeRelPath(input: string): string | undefined { + if (!input) return undefined + const raw = trim(input).replaceAll('\\', '/').replace(/^\.\//, '').replace(/^\/+/, '') + const parts: string[] = [] + for (const rawPart of raw.split('/')) { + const part = trim(rawPart) + if (!part || part === '.') continue + if (part === '..') { if (!parts.length) return undefined; parts.pop() } + else parts.push(part) + } + const result = parts.join('/') + return result && !result.includes('\0') ? result : undefined +} +function canonical(path: string): string { + try { return realpathSync(path) } catch { + const parent = dirname(path) + return parent === path ? path : resolve(canonical(parent), path.slice(parent.length + 1)) + } +} +export function resolveDoclikeTarget(href: string, file: string, root: string, sources: Record<string, string>): string | undefined { + let raw = trim(href) + if (raw.startsWith('<') && raw.endsWith('>')) raw = trim(raw.slice(1, -1)) + if (!raw || ['http://', 'https://', 'mailto:', 'tel:', 'data:', 'javascript:', '#'].some(p => raw.startsWith(p))) return + if (raw.startsWith('?p=') || raw.startsWith('/?p=') || raw.includes('&p=')) return + raw = trim(raw.split('#', 1)[0]!.split('?', 1)[0]!) + if (!raw || !(raw.endsWith('.md') || raw.endsWith('.txt') || Object.keys(sources).some(p => raw.startsWith(`${p}/`)))) return + if (raw.startsWith('/')) return pathJoin(root, raw.replace(/^\/+/, '')) + for (const [prefix, source] of Object.entries(sources)) { + if (raw.startsWith(`${prefix}/`)) { + const rel = normalizeRelPath(raw.slice(prefix.length + 1)) + return rel ? pathJoin(source, rel) : undefined + } + } + const rel = normalizeRelPath(raw) + if (!rel) return + const target = canonical(resolve(dirname(file), rel)) + const fromRoot = relative(root, target) + if (fromRoot === '..' || fromRoot.startsWith('../') || isAbsolute(fromRoot)) return + return target +} +export function checkPLinks(config: CollectionConfig, ids: Record<string, Set<string>>, includeKind = false): Issue[] { + const issues: Issue[] = [] + for (const path of markdownFiles(config.source)) for (const match of read(path).matchAll(/(?:([a-zA-Z0-9_-]+)\/)?\?p=([a-zA-Z0-9_./-]+)/g)) { + const kind = match[1] || config.kind + const id = match[2]! + if (!ids[kind]?.has(id)) issues.push({ path, message: `Broken ?p= link: ${includeKind ? `${kind}/` : ''}${id}` }) + } + return issues +} +export function checkDocLinks(source: string, root: string, sources: Record<string, string>): Issue[] { + const issues: Issue[] = [] + for (const path of markdownFiles(source)) for (const match of read(path).matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) { + const href = match[1]! + const target = resolveDoclikeTarget(href, path, root, sources) + if (target && !existsSync(target)) { + const display = target.startsWith(root + '/') ? target.slice(root.length + 1) : target + issues.push({ path, message: `Missing link target: ${href} -> ${display}` }) + } + } + return issues +} +const editorial = [ + '^\\s*#{1,6}\\s+(?:Status(?:\\s*\\([^)]*\\))?|Status and Future Work|Future Work|Follow-ups)\\s*$', + '^\\s*#{1,6}\\s+.+\\s+\\(Planned(?:[^)]*)\\)\\s*$', + '^\\s*Status:\\s*\\*\\*.+\\*\\*', + '^\\s*#{1,6}\\s+Current API(?:\\s*\\([^)]*\\))?\\s*$', + '^\\s*#{1,6}\\s+API\\s*\\((?:selected|implemented[^)]*|initial[^)]*|current[^)]*)\\)\\s*$', + '\\b(?:Implemented subset|Current supported|Implemented-subset|active expansion|current compiler subset|initial implementation)\\b', +].map(source => pattern(source, 'i')) +export function checkLines(source: string, kind: 'viewer' | 'manpage' | 'editorial'): Issue[] { + const issues: Issue[] = [] + const expressions = kind === 'editorial' ? editorial : kind === 'viewer' + ? [/(?<!\]\()(?<!\()(?<!\/)\?p=[a-zA-Z0-9_./-]+/] + : [/(?<!\[)`[A-Za-z0-9_:+.-]+` \(([137])\)/] + const message = kind === 'editorial' ? 'avoid status-style or transitional editorial framing in published docs.' + : `raw ${kind === 'viewer' ? '?p=' : 'manpage'} reference must be a markdown link.` + for (const path of markdownFiles(source)) { + let fence = false + for (const [index, line] of lines(read(path)).entries()) { + const stripped = line.replace(pattern('^\\s+'), '') + if (stripped.startsWith('```') || stripped.startsWith('~~~')) { fence = !fence; continue } + if (fence || (kind !== 'editorial' && stripped.startsWith('#'))) continue + if (expressions.some(re => re.test(line))) issues.push({ path, message: `Line ${index + 1}: ${message}` }) + } + } + return issues +} +export function reportIssues(issues: Issue[], root: string, success: string, reporter: Reporter = console, label = 'issues'): number { + if (!issues.length) { reporter.log(success); return 0 } + for (const issue of issues.slice(0, 200)) reporter.error(`${relative(root, issue.path)}: ${issue.message}`) + if (issues.length > 200) reporter.error(`... and ${issues.length - 200} more`) + reporter.error(`FAIL: ${issues.length} ${label}`) + return 1 +} +export function runSiteAudit(context: AuditContext, collection: Exclude<Collection, 'silkWiki'>, reporter: Reporter = console): number { + const config = collectionConfig(context, collection) + + return reportIssues([ + ...checkRawOutputs(config), ...checkPLinks(config, { docs: collectionIds(config) }), + ...checkDocLinks(config.source, context.outputRoot, { docs: config.source }), + ...checkLines(config.source, 'viewer'), ...checkLines(config.source, 'manpage'), ...checkLines(config.source, 'editorial'), + ], context.outputRoot, `OK: ${collection} site audit passed`, reporter) +} diff --git a/tools/audits/runtime.ts b/tools/audits/runtime.ts new file mode 100644 index 00000000..ea254979 --- /dev/null +++ b/tools/audits/runtime.ts @@ -0,0 +1,66 @@ +import { existsSync } from 'node:fs' +import { collectionConfig, markdownFiles, pathJoin, pattern, read, reportIssues, runSiteAudit, withSourceInventory } from './common.ts' +import type { AuditContext, Issue, Reporter } from './common.ts' + +const curated = new Map(['ai', 'application', 'extension', 'fs', 'hooks', 'mcp', 'notification', 'secure-storage', 'window'].map(name => [`oro:${name}`, `${name}.md`])) +const excluded = new Set(['oro:bootstrap', 'oro:external', 'oro:internal', 'oro:node', 'oro:node-esm-loader']) +const cliDocs: Record<string, string> = Object.fromEntries([ + ['oroc', 'oroc'], ...['help', 'update', 'build', 'run', 'list-devices', 'env', 'mcp', 'init', 'install-app', 'print-build-dir', 'setup', 'config', 'versions', 'version'].map(name => [`oroc ${name}`, name]), + ...['init', 'server', 'info', 'keygen', 'sign', 'verify', 'validate', 'bundle', 'extract'].map(name => [`oroc update-${name}`, `update/${name}`]), +].map(([section, path]) => [section, `cli/${path}.md`])) + +export function runtimeIssues(context: AuditContext): Issue[] { + const issues: Issue[] = [] + const docs = collectionConfig(context, 'runtime').source + const javascript = pathJoin(docs, 'javascript') + const add = (path: string, message: string): void => { issues.push({ path, message }) } + const moduleIndex = pathJoin(javascript, 'module-index.md') + if (existsSync(moduleIndex)) add(moduleIndex, 'remove this page; link JavaScript API entry points to javascript/overview.') + for (const path of markdownFiles(javascript, false).sort()) { + const text = read(path) + const name = path.slice(path.lastIndexOf('/') + 1) + if (text.includes('console.log(Object.keys(api))')) add(path, 'remove generic Object.keys(api) examples; document a real usage flow instead.') + if (!['overview.md', 'all-modules.md', ...curated.values()].includes(name) && !pattern('^\\s*##\\s+.*examples?\\b', 'im').test(text)) add(path, 'missing an Examples section.') + } + const index = context.runtimeRepo === undefined ? undefined : pathJoin(context.runtimeRepo, 'api/index.d.ts') + const specifiers = new Set(index && existsSync(index) ? [...read(index).matchAll(/^declare module ['"](oro:[^'"]+)['"]/gm)].map(match => match[1]!) : []) + const families = new Set([...specifiers].map(spec => spec.split('/')[0]!)) + if (families.size && specifiers.size) { + for (const family of [...families].filter(family => !excluded.has(family)).sort()) { + const expected = pathJoin(javascript, curated.get(family) ?? `${family.replace(/^oro:/, '')}.md`) + if (!existsSync(expected)) add(expected, `missing docs page for published module family ${family}.`) + } + for (const family of [...excluded].sort()) { + const path = pathJoin(javascript, `${family.replace(/^oro:/, '')}.md`) + if (existsSync(path)) add(path, `excluded private module family ${family} should not have a public docs page.`) + } + const allModulesPath = pathJoin(javascript, 'all-modules.md') + const allModules = read(allModulesPath) + for (const family of [...excluded].sort()) { + const escaped = family.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + if (new RegExp('`?' + escaped + '(?:/\\*|/[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)*)`?').test(allModules)) add(allModulesPath, `excluded private module family ${family} should not appear in the public module listing.`) + } + for (const spec of [...specifiers].filter(spec => !excluded.has(spec.split('/')[0]!)).sort()) { + if (!allModules.includes(spec)) add(allModulesPath, `missing published module specifier ${spec}.`) + } + } + const cli = context.runtimeRepo === undefined ? undefined : pathJoin(context.runtimeRepo, 'api/CLI.md') + const sections = new Set(cli && existsSync(cli) ? [...read(cli).matchAll(pattern('^##\\s+([^\\n]+)$', 'gm'))].map(match => match[1]!) : []) + if (sections.size) for (const section of Object.keys(cliDocs).sort()) { + const path = pathJoin(docs, cliDocs[section]!) + if (sections.has(section) && !existsSync(path)) add(path, `missing website docs page for upstream CLI section \`${section}\`.`) + } + const stalePatterns = [/oro-computer\/oro-runtime/, /oro-computer\/legacy-runtime/, pattern('\\blegacy-runtime\\b')] + for (const rel of ['index.html', 'runtime/index.html', 'runtime/docs/index.html']) { + const path = pathJoin(context.outputRoot, rel) + const text = read(path) + for (const expression of stalePatterns) if (expression.test(text)) add(path, 'stale runtime repository reference (deprecated runtime repository alias).') + } + return issues +} +export function runRuntimeAudit(context: AuditContext, reporter: Reporter = console): number { + context = withSourceInventory(context) + const rc = runSiteAudit(context, 'runtime', reporter) + if (rc !== 0) return rc + return reportIssues(runtimeIssues(context), context.outputRoot, 'OK: runtime docs audit passed', reporter, 'runtime docs issues') +} diff --git a/tools/audits/silk.ts b/tools/audits/silk.ts new file mode 100644 index 00000000..b555dda6 --- /dev/null +++ b/tools/audits/silk.ts @@ -0,0 +1,72 @@ +import { existsSync } from 'node:fs' +import { relative } from 'node:path' +import { checkDocLinks, checkRawOutputs, checkLines, checkPLinks, collectionConfig, lines, collectionIds, withSourceInventory, markdownFiles, pathJoin, pattern, read, reportIssues, trim } from './common.ts' +import type { AuditContext, Issue, Reporter } from './common.ts' + +function checkText(source: string, expression: RegExp, message: string): Issue[] { + return markdownFiles(source).filter(path => expression.test(read(path))).map(path => ({ path, message })) +} +export function runSilkAudit(context: AuditContext, reporter: Reporter = console): number { + context = withSourceInventory(context) + const configs = [collectionConfig(context, 'silk'), collectionConfig(context, 'silkWiki')] + const docs = configs[0]! + const wiki = configs[1]! + const ids = { docs: collectionIds(docs), wiki: collectionIds(wiki) } + const sources = { docs: docs.source, wiki: wiki.source, spec: pathJoin(docs.source, 'spec') } + const issues = [ + ...configs.flatMap(checkRawOutputs), + ...configs.flatMap(config => checkPLinks(config, ids, true)), + ...configs.flatMap(config => checkDocLinks(config.source, context.outputRoot, sources)), + ...configs.flatMap(config => checkText(config.source, pattern('\\bconst\\s+region\\s+arena\\b|\\bexport\\s+const\\s+region\\s+arena\\b|\\bwith\\s+arena\\b\\s*\\{|\\bfrom\\s+arena\\b|\\bglobal_arena\\b', 'i'), 'Found arena identifier in examples (use regions + neutral names).')), + ...configs.flatMap(config => checkText(config.source, pattern('\\bWorks today\\b|\\bWhat works today\\b|\\(Works today\\)|Syntax\\s*\\(Selected\\)', 'i'), "Found deprecated 'Works today' / 'Syntax (Selected)' labeling.")), + ...configs.flatMap(config => checkLines(config.source, 'viewer')), + ...configs.flatMap(config => checkLines(config.source, 'manpage')), + ...configs.flatMap(config => checkLines(config.source, 'editorial')), + ] + const spec = pathJoin(docs.source, 'spec/2026.md') + if (existsSync(spec) && pattern('\\bchecker\\.[A-Za-z0-9_]+\\b|\\bsrc/[A-Za-z0-9_./-]+|\\bc-tests/|\\btests/silk/', 'i').test(read(spec))) { + issues.push({ path: spec, message: 'Spec contains repo-internal paths/names (remove or generalize).' }) + } + return reportIssues(issues, context.outputRoot, 'OK: silk site audit passed', reporter) +} +const order = ['Description', 'Exported API', 'Examples', 'Considerations', 'Design goals', 'See also'] +const bannedHeadings = ['API', 'Current API', 'Implemented API', 'Public API', 'High-Level API', 'Intended Surface', 'Current Surface', 'Current Grammar Coverage', 'Goals', 'Design Goals', 'Future Work', 'Future work', 'Follow-ups', 'Remaining Follow-Ups', 'Important Limitations', 'Current Limitations', 'Notes and Limitations'] +const headingPatterns = ['\\bInitial Design\\b', '\\bCurrent Scope\\b', '\\bScope \\(Current\\)\\b', '\\bMVP\\b'].map(source => pattern(source, 'i')) +const statusPatterns = ['active expansion', 'current snapshot', 'Partially implemented', 'initial std wrapper', 'expansion path', '\\bMVP\\b'] +export function stdlibIssues(context: AuditContext): Issue[] { + const issues: Issue[] = [] + for (const path of markdownFiles(pathJoin(collectionConfig(context, 'silk').source, 'std'), false).sort()) { + const text = read(path) + for (const source of statusPatterns) if (pattern(source, 'i').test(text)) issues.push({ path, message: `Status/body uses banned transitional wording: ${source}` }) + const headings = lines(text).filter(line => line.startsWith('## ')).map(line => trim(line.slice(3))) + for (const heading of headings) { + for (const banned of bannedHeadings) if (heading === banned) issues.push({ path, message: `Banned heading: ${heading}` }) + for (const expression of headingPatterns) if (expression.test(heading)) issues.push({ path, message: `Banned heading pattern: ${heading}` }) + } + const positions = new Map(headings.map((heading, index) => [heading, index])) + let last = -1 + const seen: string[] = [] + for (const heading of order) { + const pos = positions.get(heading) + if (pos === undefined) continue + if (pos < last) { issues.push({ path, message: `Section order violation: ${[...seen, heading].join(' -> ')}` }); break } + last = pos + seen.push(heading) + } + } + return issues +} +export function runStdlibAudit(context: AuditContext, reporter: Reporter = console): number { + const issues = stdlibIssues(context) + if (issues.length) { + // The original stdlib audit reports workspace-relative paths on stdout, without a summary or cap. + for (const issue of issues) { + const rel = relative(context.siteRoot, issue.path) + if (rel === '..' || rel.startsWith('../')) throw new Error(`${issue.path} is not in the subpath of ${context.siteRoot}`) + reporter.log(`${rel}: ${issue.message}`) + } + return 1 + } + reporter.log('Stdlib doc audit passed.') + return 0 +} diff --git a/tools/migration/content.ts b/tools/migration/content.ts index 4c7187a6..6f6eff61 100644 --- a/tools/migration/content.ts +++ b/tools/migration/content.ts @@ -1,5 +1,5 @@ import MarkdownIt from 'markdown-it' -import { sanitizeMarkdown, sanitizeSpecMarkdown } from './public-copy.js' +import { sanitizeMarkdown, sanitizeSpecMarkdown } from './public-copy.ts' import { canonicalLink } from '../../src/lib/urls.ts' import { collections, type Collection } from '../../src/lib/collections.ts' const parser = new MarkdownIt({ html: true }) diff --git a/tools/migration/public-copy.js b/tools/migration/public-copy.ts similarity index 96% rename from tools/migration/public-copy.js rename to tools/migration/public-copy.ts index 93112f66..784f9f4d 100644 --- a/tools/migration/public-copy.js +++ b/tools/migration/public-copy.ts @@ -1,5 +1,10 @@ // Public-copy normalization, moved from the legacy viewer. Import-time only. -export function sanitizeMarkdown(markdown, { currentFile = "", kind = "docs" } = {}) { +export interface SanitizeMarkdownOptions { + currentFile?: string; + kind?: "docs" | "wiki"; +} + +export function sanitizeMarkdown(markdown: string, { currentFile = "", kind = "docs" }: SanitizeMarkdownOptions = {}): string { const banned = /(STATUS\.md|PLAN\.md|docs\/wiki\/style-guide\.md|_template-[^`\\s]+|style-guide\.md|README\.md)/; const statusLine = /^(Status:|Implementation status:)\s*/i; @@ -7,7 +12,7 @@ export function sanitizeMarkdown(markdown, { currentFile = "", kind = "docs" } = const isLanguageDoc = String(currentFile || "").startsWith("language/"); const testsHeading = /^(#{1,6})\s+Tests\b/i; - function rewriteStatusLine(line) { + function rewriteStatusLine(line: string): string | null { const m = String(line).match(/^(\s*)(Status:|Implementation status:)\s*/i); if (!m) return null; const leading = m[1] || ""; @@ -32,7 +37,7 @@ export function sanitizeMarkdown(markdown, { currentFile = "", kind = "docs" } = return leading + rest.slice(cut + cutLen); } - function rewriteOutsideCode(text) { + function rewriteOutsideCode(text: string): string { let out = text; // Headings @@ -178,8 +183,8 @@ export function sanitizeMarkdown(markdown, { currentFile = "", kind = "docs" } = const lines = String(markdown).split("\n"); const out = []; let inCode = false; - let codeLang = null; - let skipLevel = null; + let codeLang: string | null = null; + let skipLevel: number | null = null; let skipRepoFixtureList = false; let skipWikiFixtureBullets = false; @@ -307,7 +312,7 @@ export function sanitizeMarkdown(markdown, { currentFile = "", kind = "docs" } = } - function dropProposalProcess(markdown) { + function dropProposalProcess(markdown: string): string { const lines = String(markdown).split("\n"); const out = []; let i = 0; @@ -324,7 +329,7 @@ export function sanitizeMarkdown(markdown, { currentFile = "", kind = "docs" } = return out.join("\n"); } -export function sanitizeSpecMarkdown(markdown) { +export function sanitizeSpecMarkdown(markdown: string): string { // Keep the spec faithful, but remove repo-workflow prose and status framing // that does not belong in a standalone reader-facing specification. const banned = @@ -333,7 +338,7 @@ export function sanitizeSpecMarkdown(markdown) { const statusHeading = /^(#{1,6})\s+(Status|Implementation status)\s*:?\s*$/i; const testsHeading = /^(#{1,6})\s+Tests\b/i; - function rewriteStatusLine(line) { + function rewriteStatusLine(line: string): string | null { const m = String(line).match(/^(\s*)(Status:|Implementation status:)\s*/i); if (!m) return null; const leading = m[1] || ""; @@ -353,7 +358,7 @@ export function sanitizeSpecMarkdown(markdown) { return leading + rest.slice(cut + cutLen); } - function rewriteOutsideCode(text) { + function rewriteOutsideCode(text: string): string { let out = text; // Generated-spec phrasing: make the single-file edition self-contained. @@ -433,8 +438,8 @@ export function sanitizeSpecMarkdown(markdown) { const lines = md.split("\n"); const out = []; let inCode = false; - let skipLevel = null; - let codeLang = null; + let skipLevel: number | null = null; + let codeLang: string | null = null; let skipFixtureList = false; for (let line of lines) { @@ -537,7 +542,7 @@ export function sanitizeSpecMarkdown(markdown) { ); } else { // In code fences, rewrite comment text only. - const rewriteComment = (comment) => { + const rewriteComment = (comment: string): string => { const leading = (comment.match(/^\s*/) || [""])[0]; const body = comment.slice(leading.length); const rewritten = body diff --git a/tools/serve-output.ts b/tools/serve-output.ts new file mode 100644 index 00000000..d9b6a970 --- /dev/null +++ b/tools/serve-output.ts @@ -0,0 +1,67 @@ +import { createServer } from 'node:http' +import { readFile, realpath, stat } from 'node:fs/promises' +import { extname, isAbsolute, join, relative, resolve } from 'node:path' + +// Serve the already-validated artifact, without rebuilding it or injecting a client. +const root = await realpath(resolve('public')) +const types: Record<string, string> = { + '.html': 'text/html; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.map': 'application/json; charset=utf-8', + '.md': 'text/plain; charset=utf-8', + '.txt': 'text/plain; charset=utf-8', + '.xml': 'application/xml; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.webp': 'image/webp', + '.ico': 'image/x-icon', + '.woff': 'font/woff', + '.woff2': 'font/woff2', +} +function insideRoot(path: string): boolean { + const rel = relative(root, path) + return !isAbsolute(rel) && rel !== '..' && !rel.startsWith('../') +} +const server = createServer(async (req, res) => { + if (req.method !== 'GET' && req.method !== 'HEAD') { + res.writeHead(405, { Allow: 'GET, HEAD' }).end() + return + } + try { + const url = new URL(req.url || '/', 'http://127.0.0.1:9877') + const pathname = decodeURIComponent(url.pathname) + let path = await realpath(resolve(root, '.' + pathname)) + if (!insideRoot(path)) { + res.writeHead(403).end() + return + } + if ((await stat(path)).isDirectory()) { + if (!url.pathname.endsWith('/')) { + res.writeHead(301, { Location: url.pathname + '/' + url.search }).end() + return + } + path = await realpath(join(path, 'index.html')) + } + if (!insideRoot(path)) { + res.writeHead(403).end() + return + } + const body = await readFile(path) + res.writeHead(200, { + 'Content-Type': types[extname(path)] || 'application/octet-stream', + 'Content-Length': body.length, + 'Cache-Control': 'no-store', + }) + res.end(req.method === 'HEAD' ? undefined : body) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + res.writeHead(error instanceof URIError ? 400 : code === 'ENOENT' || code === 'ENOTDIR' ? 404 : 500).end() + } +}) +server.listen(9877, '127.0.0.1', () => console.log('Serving public/ at http://127.0.0.1:9877')) +for (const signal of ['SIGINT', 'SIGTERM'] as const) + process.on(signal, () => server.close()) diff --git a/tools/silk-editorial.ts b/tools/silk-editorial.ts new file mode 100644 index 00000000..05972a1e --- /dev/null +++ b/tools/silk-editorial.ts @@ -0,0 +1,192 @@ +import { basename, dirname } from 'node:path' +import { splitLines } from './source-pages.ts' +import { normalize_region_identifiers, normalize_editorial_framing } from './silk-ingestion-data.ts' + +// These ordered substitutions are the existing public-copy policy, not a new editorial pass. +function replacements(text: string, pairs: string[][], flags = 'g'): string { + for (const [pattern, replacement] of pairs) text = text.replace(new RegExp(pattern, flags), () => replacement) + return text +} +function mapLines(text: string, transform: (line: string) => string): string { + let inCode = false + return splitLines(text).map(raw => { + if (/^\s*(```|~~~)/.test(raw)) { inCode = !inCode; return raw.trimEnd() } + return inCode ? raw.trimEnd() : transform(raw) + }).join('\n') + (text.endsWith('\n') ? '\n' : '') +} +export function normalizeRegionIdentifiers(text: string): string { return replacements(text, normalize_region_identifiers) } +export function normalizeSharedMarkdown(text: string): string { + for (const [from, to] of [ + ['https://oro.computer/silk/docs/?p=', '?p='], ['https://oro.computer/silk/wiki/?p=', 'wiki/?p='], + ['?p=docs/', '?p='], ['docs/?p=', '?p='], ['?p=wiki/', 'wiki/?p='], + ]) text = text.replaceAll(from, to) + text = text.replace(/^(\s*\/\/\s*)Works today\b/gm, '$1Supported') + .replace(/^(\s*#{2,6}\s+)Syntax\s+\(Selected\)\s*$/gm, '$1Syntax') + .replace(/^(\s*#{2,6}\s+)(Example|Examples)\s+\(Works today\)(?::\s*(.+))?\s*$/gm, (_, p, label, detail) => p + label + (detail ? ': ' + detail : '')) + .replace(/^(\s*#{2,6}\s+)Works today(?::\s*(.+))?\s*$/gm, (_, p, detail) => p + 'Example' + (detail ? ': ' + detail : '')) + .replace(/^(\s*#{2,6}\s+)What works today(?:\s*\([^)]*\))?\s*$/gm, '$1Current subset') + .replace(/^Works today\b/gm, 'Current subset') + .replaceAll('examples labeled “Works today”', 'examples labeled “Example”') + return normalizeRegionIdentifiers(text) +} +function linkLabel(target: string): string { return target.split('/').at(-1)!.replaceAll('-', ' ').replaceAll('_', ' ') } +function docLinkTarget(target: string): string | undefined { + target = target.replace(/\.md$/, '') + for (const prefix of ['docs/', 'wiki/']) if (target.startsWith(prefix)) target = target.slice(prefix.length) + target = ({ 'language/README': 'start', 'usage/README': 'start', 'usage/tutorials/README': 'start', 'std/runtime-event-loop': 'std/runtime' } as Record<string, string>)[target] ?? target + return ['std/foo', 'std/foo-bar', 'man/my-app.7'].includes(target) ? undefined : target +} +export function normalizeUserFacingLinks(text: string): string { + return mapLines(text, raw => { + let line = raw.trimEnd().replace(/`?\?p=\.\.\.`?/g, 'docs links') + .replace(/`\?p=([a-zA-Z0-9_./-]+)`/g, (_, target) => `[${linkLabel(target)}](?p=${target})`) + line = line.replace(/(^|[\s:;,(])(?<!\]\()(?<!\/)\?p=([a-zA-Z0-9_./-]+)/g, (match, prefix, target) => { + const dest = docLinkTarget(target) + return dest === undefined ? match : `${prefix}[${linkLabel(dest)}](?p=${dest})` + }) + line = line.replace(/(^|[\s:;,(])(?<!\[)`([A-Za-z0-9_:+.-]+)`\s+\(([137])\)/g, (_, prefix, name: string, section) => + `${prefix}[\`${name}(${section})\`](?p=${name.startsWith('std::') ? name.replaceAll('::', '/') : `man/${name}.${section}`})`) + line = line.replace(/\[`?docs\/([a-zA-Z0-9_./-]+\.md)`?\]\([^)]+\)/g, (match, target) => { + const dest = docLinkTarget(target) + return dest === undefined ? match : `[${linkLabel(dest)}](?p=${dest})` + }) + return line.replace(/(^|[\s:;,(])`?docs\/([a-zA-Z0-9_./-]+\.md)`?/g, (match, prefix, target) => { + const dest = docLinkTarget(target) + return dest === undefined ? match : `${prefix}[${linkLabel(dest)}](?p=${dest})` + }) + }) +} +export function normalizeEditorialFraming(text: string): string { + return mapLines(text, raw => { + let line = raw + const heading = /^(\s*#{1,6}\s+)(.+?)\s*$/.exec(line) + if (heading) { + let title = heading[2].trim() + .replace(/\s+\((?:Initial\s+)?(?:Current\s+)?(?:Implemented|supported|compiler|initial|selected|planned|current|implementation status)[^)]*\)/gi, '') + .replace(/\s+\(Planned[^)]*\)/gi, '') + if (/^(?:Status(?:\s*\([^)]*\))?|Implementation Status(?:\s*\([^)]*\))?|Status and Future Work|Future Work|Follow-ups)$/i.test(title)) title = 'Notes' + else if (/^Notes and Limitations$/i.test(title)) title = 'Considerations' + else if (/^(?:Current API(?:\s*\([^)]*\))?|Implemented API|Public API|API\s*\([^)]*\)|API\s*\(current\))$/i.test(title)) title = 'Exported API' + else if (/^(?:Current Implemented Subset|Implemented Subset|Implemented subset)$/i.test(title)) title = 'Notes' + line = heading[1] + title.replace(/\s{2,}/g, ' ').trim() + } + return replacements(line, normalize_editorial_framing, 'gi').replace(/\s{2,}/g, ' ').trimEnd() + }) +} +export function normalizeTrailingNewlines(text: string): string { return text.trimEnd() + '\n' } +export function moveSectionToEnd(text: string, heading: string): string { + const section: string[] = [], out: string[] = [] + let inCode = false, capture = false, captureLevel = 0 + for (const raw of splitLines(text)) { + if (/^\s*(```|~~~)/.test(raw)) inCode = !inCode + if (!inCode) { + const m = /^(#{1,6})\s+(.+?)\s*$/.exec(raw) + if (m) { + if (capture && m[1].length <= captureLevel) capture = false + if (!capture && m[2].trim().toLowerCase() === heading.toLowerCase()) { capture = true; captureLevel = m[1].length } + } + } + ;(capture ? section : out).push(raw.trimEnd()) + } + if (!section.length) return text + while (out.length && !out.at(-1)!.trim()) out.pop() + while (section.length && !section.at(-1)!.trim()) section.pop() + return [...out, '', ...section].join('\n') + '\n' +} +export function normalizeHeadingsForContext(path: string, text: string): string { + text = text.replaceAll('`docs/language/mutability.md`', 'the mutability docs') + .replaceAll('`docs/language/literals-duration.md`', 'the duration literal docs') + .replaceAll('?p=std/runtime-event-loop', '?p=std/runtime') + if (path.includes('/wiki/source/')) text = text.replace(/\]\(\?p=(compiler|man|usage|guides)\//g, '](../docs/?p=$1/') + if (basename(dirname(path)) === 'std') { + for (const [from, to] of [ + ['API', 'Exported API'], ['High-Level API', 'Exported API'], ['Intended Surface', 'Exported API'], ['Current Surface', 'Exported API'], + ['Current Grammar Coverage', 'Grammar coverage'], ['Goals', 'Design goals'], ['Design Goals', 'Design goals'], + ['Important Limitations', 'Considerations'], ['Remaining Follow-Ups', 'Considerations'], + ]) text = text.replace(new RegExp('^## ' + from + '$', 'gm'), '## ' + to) + text = text.replace(/^## Implemented (`std::interfaces` surface)$/gm, '## $1') + .replaceAll('current API surface', 'exported API surface').replaceAll('API surface (current):', 'API surface:') + text = moveSectionToEnd(text, 'Design goals') + } + if (basename(dirname(path)) === 'language') { + text = text.replace(/^## Goals$/gm, '## Semantics').replace(/^## Design Goals$/gm, '## Model') + .replace(/^### Important Limitations$/gm, '### Considerations') + if (basename(path) === 'flow-for.md') text = text.replaceAll('element.\n\n: integer range iteration', 'element.\n\nSupported forms include integer range iteration') + .replace(/\n## Semantics\n\n- Provide a readable, structured loop construct for iteration\.\n- Avoid .+? hidden allocation\.\n\n## Surface Syntax/gs, '\n## Surface Syntax') + } + return text +} +export function sanitizeWikiMarkdown(_rel: string, text: string): string { + text = normalizeSharedMarkdown(text) + const out: string[] = [] + let inCode = false + for (const raw of splitLines(text)) { + if (raw.trimStart().startsWith('```')) { inCode = !inCode; out.push(raw.trimEnd()); continue } + if (inCode) { out.push(raw.trimEnd()); continue } + if (/^\s*[-*+]\s*(End-to-end support snapshot|Implemented-subset notes)\s*:\s*`?(STATUS|PLAN)\.md`?\s*$/i.test(raw)) continue + let line = raw.replace(/^(\s*[-*+]\s+)Relevant fixtures:\s*/gi, '$1Fixtures: ') + .replace(/^\s*Status:\s*implemented for the current front-end \+\s*native backend subset\.\s*$/gi, 'Implemented in the reference compiler (front-end + native backend subset).') + for (const name of ['STATUS', 'PLAN']) line = line.replace(new RegExp('\\s+(?:and|&)\\s+`?' + name + '\\.md`?\\s*$', 'gi'), '') + .replace(new RegExp('`?' + name + '\\.md`?', 'gi'), '') + line = line.replace(/\s+(?:and|&)\s*$/gi, '') + const leading = /^\s*/.exec(line)![0] + line = (leading + line.slice(leading.length).replace(/[ \t]{2,}/g, ' ')).trimEnd() + if (/^\s*[-*+]\s*[^A-Za-z0-9`]*\s*$/.test(line)) continue + out.push(line) + } + return normalizeTrailingNewlines(normalizeUserFacingLinks(normalizeEditorialFraming(out.join('\n') + (text.endsWith('\n') ? '\n' : '')))) +} +function dropNamedSection(text: string, heading: string): string { + const out: string[] = [] + let inCode = false, skipLevel: number | undefined + for (const raw of splitLines(text)) { + if (raw.trimStart().startsWith('```')) { + inCode = !inCode + if (skipLevel === undefined) out.push(raw.trimEnd()) + continue + } + if (!inCode && skipLevel !== undefined) { + const m = /^(#{1,6})\s+(.+?)\s*$/.exec(raw) + if (!m || m[1].length > skipLevel) continue + skipLevel = undefined + } + if (!inCode) { + const m = /^(#{1,6})\s+(.+?)\s*$/.exec(raw) + if (m && m[2].trim().toLowerCase() === heading.trim().toLowerCase()) { skipLevel = m[1].length; continue } + } + if (skipLevel === undefined) out.push(raw.trimEnd()) + } + return out.join('\n') + (text.endsWith('\n') ? '\n' : '') +} +export function sanitizeDocsMarkdown(rel: string, text: string): string { + text = normalizeSharedMarkdown(text).replaceAll('`STATUS.md`', '[implementation status](?p=compiler/implementation-status)') + .replaceAll('[implementation status](?p=compiler/implementation-status) (implementation status)', '[implementation status](?p=compiler/implementation-status)') + .replace(/\s*\(tracked[^)]*`PLAN\.md`\)/gi, '') + text = dropNamedSection(dropNamedSection(text, 'Arenas'), 'Tests') + if (rel.startsWith('std/')) text = text.replace(/^Status:\s*\*\*[^*]+\*\*\.\s*/gm, '').replace(/^Status:\s*\*\*[^*]+\*\*\.\s*$/gm, '') + .replace(/^##\s+Current API\s*$/gm, '## Exported API').replace(/^##\s+Implemented API\s*$/gm, '## Exported API').replace(/^##\s+Public API\s*$/gm, '## Exported API') + .replace(/^##\s+(Future Work|Future work|Follow-ups|Current Limitations|Notes and Limitations)\s*$/gm, '## Considerations') + .replace(/^##\s+Current Scope\s*$/gm, '## Considerations') + .replace(/^##\s+(.+?)\s+\((?:Initial Design|MVP|Current)\)\s*$/gm, '## $1') + .replace(/active expansion/gi, 'current module surface').replace(/current snapshot/gi, 'current implementation') + .replace(/Partially implemented/gi, 'Implemented').replace(/initial std wrapper/gi, 'stdlib wrapper').replace(/\bMVP\b/g, 'baseline') + if (rel === 'spec/2026.md') text = dropNamedSection(text, 'Silk Proposal Process (TC39-Inspired)') + .replaceAll('// Works today', '// Supported').replaceAll('checker.checkModuleSetWithImports', 'the module-set import helper') + .replace(/CLI output:\s*`+silk` CLI`?\s+and\s+`+silk`\s+\(1\)\s+—\s+Silk Language Compiler`?\s+\(`silk --version`\)/g, + 'CLI output: the `silk` CLI and [`silk(1)`](?p=man/silk.1) (`silk --version`)') + if (rel === 'compiler/vendored-deps.md') text = text.replace('# Built-In Dependencies\n', '# Built-In Dependencies (Legacy Link)\n') + const out: string[] = [] + let inCode = false + for (const raw of splitLines(text)) { + if (raw.trimStart().startsWith('```')) { inCode = !inCode; out.push(raw.trimEnd()); continue } + if (inCode) { out.push(raw.trimEnd()); continue } + const parts = raw.split('`') + for (let i = 0; i < parts.length; i += 2) parts[i] = parts[i] + .replace(/\bThe repository\b/g, 'The Silk compiler repository').replace(/\bthe repository\b/g, 'the Silk compiler repository') + .replace(/\bThis repository\b/g, 'The Silk compiler repository').replace(/\bthis repository\b/g, 'the Silk compiler repository') + .replace(/\brepo dependency workflow\b/gi, 'Silk compiler repository’s vendored dependency workflow') + .replace(/\bwhat works today\b/gi, 'current implementation notes') + out.push(parts.join('`').trimEnd()) + } + return normalizeTrailingNewlines(normalizeUserFacingLinks(normalizeEditorialFraming(out.join('\n') + (text.endsWith('\n') ? '\n' : '')))) +} diff --git a/tools/silk-ingestion-data.ts b/tools/silk-ingestion-data.ts new file mode 100644 index 00000000..13c0324c --- /dev/null +++ b/tools/silk-ingestion-data.ts @@ -0,0 +1,231 @@ +// Original ownership policy and ordered editorial replacements. +export const EXCLUDE_BASENAMES = [ + "PLAN.md", + "README.md", + "STATUS.md", + "TEMPLATE.md", + "_template-language.md", + "_template-stdlib.md", + "arenas.md", + "llms.txt", + "style-guide.md" +] + +export const KEEP_DOCS_PREFIXES = [ + "guides/" +] + +export const KEEP_DOCS_FILES = [ + "compiler/backend-wasm.md", + "compiler/implementation-status.md", + "compiler/libsilk-quickstart.md", + "compiler/testing-strategy.md", + "compiler/zig-api.md", + "start.md", + "usage/cli-examples.md", + "usage/editor-coc-nvim.md", + "usage/editor-ctags.md", + "usage/editor-textmate.md", + "usage/editor-vim.md", + "usage/getting-started.md", + "usage/github-linguist.md", + "usage/howto-custom-stdlib-root.md", + "usage/howto-run-wasi-node.md", + "usage/tutorials/01-first-program.md", + "usage/tutorials/02-structs-and-impls.md", + "usage/tutorials/03-arrays-and-slices.md", + "usage/tutorials/04-filesystem.md", + "usage/tutorials/05-concurrency.md", + "usage/tutorials/06-async-io-streams-abort.md", + "usage/tutorials/07-formal-silk.md" +] + +export const KEEP_WIKI_FILES = [ + "start.md" +] + +export const normalize_region_identifiers = [ + [ + "\\bexport const region global_arena\\b", + "export const region global_region_buf" + ], + [ + "\\bconst region arena\\b", + "const region region_buf" + ], + [ + "\\bwith arena\\b", + "with region_buf" + ], + [ + "\\bfrom arena\\b", + "from region_buf" + ], + [ + "\\bglobal_arena\\b", + "global_region_buf" + ], + [ + "\\barena\\[", + "region_buf[" + ], + [ + "`arena`", + "`region_buf`" + ], + [ + "`global_arena`", + "`global_region_buf`" + ] +] + +export const normalize_editorial_framing = [ + [ + "\\bStatus:\\s*\\*\\*([^*]+)\\*\\*\\.\\s*", + "" + ], + [ + "\\bStatus:\\s*\\*\\*([^*]+)\\*\\*\\s*", + "" + ], + [ + "\\bStatus:\\s*", + "" + ], + [ + "\\bImplementation status\\s*\\([^)]*\\)\\s*:\\s*", + "" + ], + [ + "\\bImplementation status\\s*:\\s*", + "" + ], + [ + "Returning readers typically want the “Implementation Status” section near the top", + "Returning readers typically want the notes near the top" + ], + [ + "see “Implementation Status” below", + "see the notes below" + ], + [ + "Implementation Status” sections", + "notes sections" + ], + [ + "\\bCurrent supported contexts include\\b", + "Supported contexts include" + ], + [ + "\\bcurrent supported\\b", + "supported" + ], + [ + "\\bCurrent compiler subset restriction\\b", + "Restriction" + ], + [ + "\\bthe current compiler subset\\b", + "Silk currently" + ], + [ + "\\bThe current compiler subset\\b", + "Silk currently" + ], + [ + "\\bcurrent compiler subset\\b", + "current implementation" + ], + [ + "\\bCurrent subset\\b", + "Supported forms" + ], + [ + "\\bcurrent subset\\b", + "supported forms" + ], + [ + "\\bInitial implementation target\\b", + "Implementation target" + ], + [ + "\\binitial implementation target\\b", + "implementation target" + ], + [ + "\\binitial implementation\\b", + "implementation" + ], + [ + "\\bInitial implementation\\b", + "Implementation" + ], + [ + "\\bImplemented-subset notes\\b", + "Notes" + ], + [ + "\\bImplemented-subset details\\b", + "Reference details" + ], + [ + "\\bImplemented-subset\\b", + "Reference" + ], + [ + "\\bImplemented subset notes\\b", + "Notes" + ], + [ + "\\bimplemented subset notes\\b", + "notes" + ], + [ + "\\bImplemented subset\\b", + "Supported forms" + ], + [ + "\\bimplemented subset\\b", + "supported forms" + ], + [ + "\\bImplemented Subset\\b", + "Supported Forms" + ], + [ + "\\bInitial Implemented Subset\\b", + "Supported Forms" + ], + [ + "\\bCurrent Implemented Subset\\b", + "Supported Forms" + ], + [ + "\\bexamples that exercise the supported forms\\b", + "examples" + ], + [ + "\\bExamples that exercise the supported forms\\b", + "Examples" + ], + [ + "\\bactive expansion\\b", + "current module surface" + ], + [ + "\\bPartially implemented\\b", + "Implemented" + ], + [ + "\\bpartially implemented\\b", + "implemented" + ], + [ + "\\bin progress\\b", + "available" + ], + [ + "\\bIn progress\\b", + "Available" + ] +] diff --git a/tools/site_audit_common.py b/tools/site_audit_common.py deleted file mode 100644 index 3a5e19a5..00000000 --- a/tools/site_audit_common.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import json -import re -import sys -from dataclasses import dataclass -from pathlib import Path - - -P_LINK_RE = re.compile(r"(?:(?P<kind>[a-zA-Z0-9_-]+)/)?\?p=(?P<id>[a-zA-Z0-9_./-]+)") -MD_LINK_RE = re.compile(r"\[[^\]]*\]\(([^)]+)\)") -RAW_VIEWER_REF_RE = re.compile(r"(?<!\]\()(?<!\()(?<!/)\?p=[a-zA-Z0-9_./-]+") -RAW_MANPAGE_REF_RE = re.compile(r"(?<!\[)`[A-Za-z0-9_:+.-]+` \(([137])\)") -STATUS_HEADING_RE = re.compile( - r"^\s*#{1,6}\s+(?:Status(?:\s*\([^)]*\))?|Status and Future Work|Future Work|Follow-ups)\s*$", - re.IGNORECASE, -) -PLANNED_HEADING_RE = re.compile(r"^\s*#{1,6}\s+.+\s+\(Planned(?:[^)]*)\)\s*$", re.IGNORECASE) -STATUS_BANNER_RE = re.compile(r"^\s*Status:\s*\*\*.+\*\*", re.IGNORECASE) -CURRENT_API_HEADING_RE = re.compile(r"^\s*#{1,6}\s+Current API(?:\s*\([^)]*\))?\s*$", re.IGNORECASE) -SELECTED_API_HEADING_RE = re.compile( - r"^\s*#{1,6}\s+API\s*\((?:selected|implemented[^)]*|initial[^)]*|current[^)]*)\)\s*$", - re.IGNORECASE, -) -TRANSITIONAL_PHRASE_RE = re.compile( - r"\b(?:Implemented subset|Current supported|Implemented-subset|active expansion|current compiler subset|initial implementation)\b", - re.IGNORECASE, -) - -SKIP_PREFIXES = ("http://", "https://", "mailto:", "tel:", "data:", "javascript:", "#") - - -@dataclass(frozen=True) -class Issue: - path: Path - message: str - - -@dataclass(frozen=True) -class SiteAuditConfig: - project_root: Path - docs_source: Path - docs_index: Path - missing_index_message: str - success_message: str - - @property - def repo_root(self) -> Path: - return self.project_root.parent - - -def load_ids(index_path: Path) -> set[str]: - data = json.loads(index_path.read_text(encoding="utf-8")) - ids: set[str] = set() - for sec in data.get("sections", []): - for item in sec.get("items", []): - ids.add(item["id"]) - return ids - - -def load_index_files(index_path: Path) -> list[str]: - data = json.loads(index_path.read_text(encoding="utf-8")) - files: list[str] = [] - for sec in data.get("sections", []): - for item in sec.get("items", []): - files.append(item["file"]) - return files - - -def normalize_rel_path(input_path: str) -> str | None: - if not input_path: - return None - - raw_path = str(input_path).strip().replace("\\", "/").removeprefix("./").lstrip("/") - parts: list[str] = [] - for raw_part in raw_path.split("/"): - part = raw_part.strip() - if not part or part == ".": - continue - if part == "..": - if not parts: - return None - parts.pop() - continue - parts.append(part) - - normalized = "/".join(parts) - if not normalized or "\0" in normalized: - return None - return normalized - - -def resolve_doclike_target_from_roots( - href: str, - current_file: Path, - repo_root: Path, - source_roots: dict[str, Path], -) -> Path | None: - raw = href.strip() - if not raw: - return None - if raw.startswith("<") and raw.endswith(">"): - raw = raw[1:-1].strip() - if not raw or raw.startswith(SKIP_PREFIXES): - return None - if raw.startswith("?p=") or raw.startswith("/?p=") or "&p=" in raw: - return None - - raw = raw.split("#", 1)[0].split("?", 1)[0].strip() - if not raw: - return None - - looks_like_doc = raw.endswith(".md") or raw.endswith(".txt") or any( - raw.startswith(f"{prefix}/") for prefix in source_roots - ) - if not looks_like_doc: - return None - - if raw.startswith("/"): - return repo_root / raw.lstrip("/") - - for prefix, root in source_roots.items(): - if raw.startswith(f"{prefix}/"): - rel = normalize_rel_path(raw.removeprefix(f"{prefix}/")) - return (root / rel) if rel else None - - rel = normalize_rel_path(raw) - if not rel: - return None - - resolved = (current_file.parent / rel).resolve() - try: - resolved.relative_to(repo_root) - except ValueError: - return None - return resolved - - -def resolve_doclike_target(config: SiteAuditConfig, href: str, current_file: Path) -> Path | None: - return resolve_doclike_target_from_roots( - href=href, - current_file=current_file, - repo_root=config.repo_root, - source_roots={"docs": config.docs_source}, - ) - - -def iter_auditable_lines(md: Path) -> list[tuple[int, str]]: - text = md.read_text(encoding="utf-8") - out: list[tuple[int, str]] = [] - in_fence = False - for lineno, line in enumerate(text.splitlines(), start=1): - stripped = line.lstrip() - if stripped.startswith("```") or stripped.startswith("~~~"): - in_fence = not in_fence - continue - if in_fence or stripped.startswith("#"): - continue - out.append((lineno, line)) - return out - - -def check_index_files_exist_at(index_path: Path, source_root: Path) -> list[Issue]: - issues: list[Issue] = [] - for file_name in load_index_files(index_path): - target = source_root / file_name - if not target.exists(): - issues.append(Issue(index_path, f"Index refers to missing file: {file_name}")) - return issues - - -def check_index_files_exist(config: SiteAuditConfig) -> list[Issue]: - return check_index_files_exist_at(config.docs_index, config.docs_source) - - -def check_p_links_in_tree( - source_root: Path, - default_kind: str, - ids_by_kind: dict[str, set[str]], - *, - include_kind_in_message: bool = False, -) -> list[Issue]: - issues: list[Issue] = [] - for md in source_root.rglob("*.md"): - text = md.read_text(encoding="utf-8") - for match in P_LINK_RE.finditer(text): - target_kind = match.group("kind") or default_kind - target_id = match.group("id") - ids = ids_by_kind.get(target_kind, set()) - if target_id not in ids: - message = f"Broken ?p= link: {target_kind}/{target_id}" if include_kind_in_message else f"Broken ?p= link: {target_id}" - issues.append(Issue(md, message)) - return issues - - -def check_p_links(config: SiteAuditConfig, ids: set[str]) -> list[Issue]: - return check_p_links_in_tree(config.docs_source, "docs", {"docs": ids}) - - -def check_doclike_links_in_tree( - source_root: Path, - repo_root: Path, - source_roots: dict[str, Path], -) -> list[Issue]: - issues: list[Issue] = [] - for md in source_root.rglob("*.md"): - text = md.read_text(encoding="utf-8") - for match in MD_LINK_RE.finditer(text): - href = match.group(1) - target = resolve_doclike_target_from_roots(href, md, repo_root, source_roots) - if not target: - continue - if not target.exists(): - try: - rel = target.relative_to(repo_root) - except ValueError: - rel = target - issues.append(Issue(md, f"Missing link target: {href} -> {rel}")) - return issues - - -def check_doclike_links(config: SiteAuditConfig) -> list[Issue]: - return check_doclike_links_in_tree( - config.docs_source, - config.repo_root, - {"docs": config.docs_source}, - ) - - -def check_no_raw_viewer_refs_in_tree(source_root: Path) -> list[Issue]: - issues: list[Issue] = [] - for md in source_root.rglob("*.md"): - for lineno, line in iter_auditable_lines(md): - if RAW_VIEWER_REF_RE.search(line): - issues.append(Issue(md, f"Line {lineno}: raw ?p= reference must be a markdown link.")) - return issues - - -def check_no_raw_viewer_refs(config: SiteAuditConfig) -> list[Issue]: - return check_no_raw_viewer_refs_in_tree(config.docs_source) - - -def check_no_raw_manpage_refs_in_tree(source_root: Path) -> list[Issue]: - issues: list[Issue] = [] - for md in source_root.rglob("*.md"): - for lineno, line in iter_auditable_lines(md): - if RAW_MANPAGE_REF_RE.search(line): - issues.append(Issue(md, f"Line {lineno}: raw manpage reference must be a markdown link.")) - return issues - - -def check_no_raw_manpage_refs(config: SiteAuditConfig) -> list[Issue]: - return check_no_raw_manpage_refs_in_tree(config.docs_source) - - -def check_no_editorial_status_framing_in_tree(source_root: Path) -> list[Issue]: - issues: list[Issue] = [] - for md in source_root.rglob("*.md"): - text = md.read_text(encoding="utf-8") - in_fence = False - for lineno, line in enumerate(text.splitlines(), start=1): - stripped = line.lstrip() - if stripped.startswith("```") or stripped.startswith("~~~"): - in_fence = not in_fence - continue - if in_fence: - continue - if ( - STATUS_HEADING_RE.search(line) - or PLANNED_HEADING_RE.search(line) - or STATUS_BANNER_RE.search(line) - or CURRENT_API_HEADING_RE.search(line) - or SELECTED_API_HEADING_RE.search(line) - or TRANSITIONAL_PHRASE_RE.search(line) - ): - issues.append( - Issue( - md, - f"Line {lineno}: avoid status-style or transitional editorial framing in published docs.", - ) - ) - return issues - - -def check_no_editorial_status_framing(config: SiteAuditConfig) -> list[Issue]: - return check_no_editorial_status_framing_in_tree(config.docs_source) - - -def run_site_audit(config: SiteAuditConfig) -> int: - if not config.docs_index.exists(): - print(config.missing_index_message, file=sys.stderr) - return 2 - - ids = load_ids(config.docs_index) - issues: list[Issue] = [] - issues += check_index_files_exist(config) - issues += check_p_links(config, ids) - issues += check_doclike_links(config) - issues += check_no_raw_viewer_refs(config) - issues += check_no_raw_manpage_refs(config) - issues += check_no_editorial_status_framing(config) - - if issues: - for issue in issues[:200]: - rel = issue.path.relative_to(config.repo_root) - print(f"{rel}: {issue.message}", file=sys.stderr) - if len(issues) > 200: - print(f"... and {len(issues) - 200} more", file=sys.stderr) - print(f"FAIL: {len(issues)} issues", file=sys.stderr) - return 1 - - print(config.success_message) - return 0 diff --git a/tools/source-pages.py b/tools/source-pages.py deleted file mode 100644 index 2ca65a3a..00000000 --- a/tools/source-pages.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Bridge manual upstream generators to committed DOMStack Markdown pages. - -A temporary flat source tree lets existing editorial logic retain ownership rules. -Only the final Node importer writes page.md files and metadata to the website. -""" -from __future__ import annotations -import subprocess -from pathlib import Path - - -def stage_collection(site: Path, collection: str, dest: Path) -> None: - subprocess.run(['node', str(site / 'tools' / 'import-public.ts'), '--export', - collection, str(dest), str(site)], check=True, cwd=site) - - -def import_collection(site: Path, collection: str, source: Path) -> None: - subprocess.run(['node', str(site / 'tools' / 'import-public.ts'), - collection, str(source), str(site)], check=True, cwd=site) - - -def preserve_fences(text: str, transform) -> str: - import re - fences = [] - lines = text.splitlines(keepends=True) - protected = [] - i = 0 - while i < len(lines): - opening = re.match(r'^ {0,3}(`{3,}|~{3,})([^\n]*)', lines[i]) - if not opening or (opening[1][0] == '`' and '`' in opening[2]): - protected.append(lines[i]) - i += 1 - continue - start = i - marker = opening[1] - i += 1 - while i < len(lines): - closing = re.match(r'^ {0,3}(' + re.escape(marker[0]) + r'{'+str(len(marker))+r',})[ \t]*$', lines[i].rstrip('\n')) - i += 1 - if closing: - break - fences.append(''.join(lines[start:i])) - protected.append(f'OROFENCE{len(fences)-1}END\n') - return re.sub(r'OROFENCE(\d+)END\n?', lambda m: fences[int(m.group(1))], transform(''.join(protected))) diff --git a/tools/source-pages.ts b/tools/source-pages.ts new file mode 100644 index 00000000..1be1ba5c --- /dev/null +++ b/tools/source-pages.ts @@ -0,0 +1,74 @@ +import { readFile, writeFile, mkdir, readdir, stat } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import type { Collection } from '#lib/collections.ts' + +export async function stageCollection(collection: Collection, dest: string, site: string): Promise<void> { + const { exportCollection } = await import('./import-public.ts') + await exportCollection(collection, dest, site) +} +export async function importCollection(collection: Collection, source: string, site: string): Promise<void> { + const { importCollection: importPublicCollection } = await import('./import-public.ts') + await importPublicCollection(collection, source, site) +} + +// Python's text readers normalize universal newlines before editorial processing. +export async function readText(path: string): Promise<string> { + return (await readFile(path, 'utf8')).replace(/\r\n?/g, '\n') +} +export async function writeTextIfChanged(path: string, text: string): Promise<boolean> { + let previous: string | undefined + try { previous = await readText(path) } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + if (previous === text) return false + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, text) + return true +} +export async function files(root: string): Promise<string[]> { + try { + const entries = await readdir(root, { recursive: true, withFileTypes: true }) + const paths: string[] = [] + for (const entry of entries) { + const path = join(entry.parentPath, entry.name) + if (entry.isFile()) paths.push(path) + else if (entry.isSymbolicLink()) { + try { if ((await stat(path)).isFile()) paths.push(path) } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + } + } + } + return paths.sort() + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return [] + throw error + } +} +export function splitLines(text: string): string[] { + if (!text) return [] + const lines = text.split(/\r\n|[\n\r\v\f\x1c-\x1e\x85\u2028\u2029]/) + if (lines.at(-1) === '') lines.pop() + return lines +} +export function preserveFences(text: string, transform: (text: string) => string): string { + const fences: string[] = [] + const lines = text.match(/[^\n\r\v\f\x1c-\x1e\x85\u2028\u2029]*(?:\r\n|[\n\r\v\f\x1c-\x1e\x85\u2028\u2029])|[^\n\r\v\f\x1c-\x1e\x85\u2028\u2029]+$/g) ?? [] + const protectedLines: string[] = [] + // Avoid interpreting authored text as one of our placeholders. + let token = 'OROFENCE' + while (text.includes(token)) token += 'X' + for (let i = 0; i < lines.length;) { + const opening = /^ {0,3}(`{3,}|~{3,})([^\n]*)/.exec(lines[i]) + if (!opening || (opening[1][0] === '`' && opening[2].includes('`'))) { + protectedLines.push(lines[i++]); continue + } + const start = i++ + const closing = new RegExp('^ {0,3}' + opening[1][0] + '{' + opening[1].length + ',}[ \\t]*$') + while (i < lines.length) { + if (closing.test(lines[i++].replace(/\n$/, ''))) break + } + fences.push(lines.slice(start, i).join('')) + protectedLines.push(`${token}${fences.length - 1}END\n`) + } + return transform(protectedLines.join('')).replace(new RegExp(token + '(\\d+)END\\n?', 'g'), (_, index) => fences[Number(index)]) +} diff --git a/tsconfig.json b/tsconfig.json index 3a922577..82abd853 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,6 +22,8 @@ "include": [ "src/**/*.ts", "tools/**/*.ts", + "runtime/tools/**/*.ts", + "silk/tools/**/*.ts", "tests/**/*.ts", "playwright.config.ts" ] diff --git a/virtnosis/tools/audit-site.py b/virtnosis/tools/audit-site.py deleted file mode 100644 index d9f18eba..00000000 --- a/virtnosis/tools/audit-site.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import sys -import os -from pathlib import Path - - -SITE_ROOT = Path(__file__).resolve().parents[2] -VIRTNOSIS_ROOT = Path(os.environ.get("ORO_SITE_OUTPUT", str(SITE_ROOT / "public"))) / "virtnosis" -REPO_ROOT = VIRTNOSIS_ROOT.parent - -sys.path.insert(0, str(SITE_ROOT / "tools")) - -from site_audit_common import SiteAuditConfig, run_site_audit - - -def main() -> int: - return run_site_audit( - SiteAuditConfig( - project_root=VIRTNOSIS_ROOT, - docs_source=VIRTNOSIS_ROOT / "docs" / "source", - docs_index=VIRTNOSIS_ROOT / "docs" / "index.json", - missing_index_message="Missing Virtnosis docs/index.json; run build scripts first.", - success_message="OK: virtnosis site audit passed", - ) - ) - - -if __name__ == "__main__": - raise SystemExit(main())