Serve runtime-observed output shapes for schemaless tools

e2e/scenarios/shape-memory.test.ts

@@ -0,0 +1,178 @@
+// Cross-target: muscle memory — runtime-observed output shapes. Most OpenAPI
+// operations declare no response schema, so `tools.describe.tool()` used to
+// render `data: unknown` forever and the model had to guess response shapes.
+// This journey proves the warm path end to end through public surfaces only:
+// a schemaless tool describes as `unknown`, one real invocation against a live
+// upstream teaches the shape, and the very next describe serves a real
+// TypeScript type marked as observed.
+import { randomBytes } from "node:crypto";
+import { createServer } from "node:http";
+
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+import { composePluginApi } from "@executor-js/api/server";
+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
+
+import { scenario } from "../src/scenario";
+import { Api, Target } from "../src/services";
+
+const api = composePluginApi([openApiHttpPlugin()] as const);
+
+/** One GET operation whose 200 declares no response schema — the shape the
+ *  model would otherwise have to guess. */
+const issuesSpec = JSON.stringify({
+  openapi: "3.0.3",
+  info: { title: "Issues API", version: "1.0.0" },
+  paths: {
+    "/issues": {
+      get: {
+        operationId: "listIssues",
+        summary: "List issues",
+        responses: { "200": { description: "issues" } },
+      },
+    },
+  },
+});
+
+/** A live upstream for the single invocation that teaches the shape. */
+const serveIssuesFixture = Effect.acquireRelease(
+  Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => {
+    const server = createServer((_request, response) => {
+      response.writeHead(200, { "content-type": "application/json" });
+      response.end(
+        JSON.stringify({
+          issues: [
+            { id: 1, title: "first", open: true },
+            { id: 2, title: "second", open: false },
+          ],
+          total: 2,
+        }),
+      );
+    });
+    server.listen(0, "127.0.0.1", () => {
+      const addressInfo = server.address();
+      const port = typeof addressInfo === "object" && addressInfo !== null ? addressInfo.port : 0;
+      resume(
+        Effect.succeed({
+          url: `http://127.0.0.1:${port}`,
+          close: () => {
+            server.close();
+            server.closeAllConnections();
+          },
+        }),
+      );
+    });
+  }),
+  (fixture) => Effect.sync(fixture.close),
+);
+
+const describeCode = (slug: string) => `
+const details = await tools.describe.tool({ path: "${slug}.org.main.issues.listIssues" });
+return {
+  outputTypeScript: details.outputTypeScript ?? null,
+  note: details.outputTypeScriptNote ?? null,
+  error: details.error ?? null,
+};
+`;
+
+type DescribeOutcome = {
+  readonly outputTypeScript: string | null;
+  readonly note: string | null;
+  readonly error: unknown;
+};
+
+scenario(
+  "Muscle memory · a schemaless tool's observed output shape reaches describe",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const { client: makeApiClient } = yield* Api;
+      const identity = yield* target.newIdentity();
+      const client = yield* makeApiClient(api, identity);
+      const slug = IntegrationSlug.make(`shape_memory_${randomBytes(4).toString("hex")}`);
+      const upstream = yield* serveIssuesFixture;
+
+      yield* Effect.ensuring(
+        Effect.gen(function* () {
+          yield* client.openapi.addSpec({
+            payload: {
+              spec: { kind: "blob", value: issuesSpec },
+              slug,
+              baseUrl: upstream.url,
+              authenticationTemplate: [
+                {
+                  slug: "apiKey",
+                  type: "apiKey",
+                  headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] },
+                },
+              ],
+            },
+          });
+          yield* client.connections.create({
+            payload: {
+              owner: "org",
+              name: ConnectionName.make("main"),
+              integration: slug,
+              template: AuthTemplateSlug.make("apiKey"),
+              value: `key_${randomBytes(8).toString("hex")}`,
+            },
+          });
+
+          const describe = Effect.gen(function* () {
+            const executed = yield* client.executions.execute({
+              payload: { code: describeCode(String(slug)), autoApprove: true },
+            });
+            expect(executed.status, executed.text).toBe("completed");
+            return JSON.parse(executed.text) as DescribeOutcome;
+          });
+
+          // Cold: no declared response schema — the model sees unknown.
+          const cold = yield* describe;
+          expect(cold.error, "the tool resolves").toBeNull();
+          expect(cold.outputTypeScript, "cold describe has no shape").toContain("data: unknown;");
+          expect(cold.note, "cold describe carries no provenance note").toBeNull();
+
+          // One real call against the live upstream teaches the shape.
+          const invoked = yield* client.executions.execute({
+            payload: {
+              code: `
+const result = await tools.${slug}.org.main.issues.listIssues({});
+return { ok: result.ok };
+`,
+              autoApprove: true,
+            },
+          });
+          expect(invoked.status, invoked.text).toBe("completed");
+          expect(JSON.parse(invoked.text), "the teaching call succeeded").toEqual({ ok: true });
+
+          // Warm: the observed shape is served, marked as observed.
+          const warm = yield* describe;
+          expect(warm.outputTypeScript, "warm describe serves the observed shape").toContain(
+            "issues",
+          );
+          expect(warm.outputTypeScript, "field types come from the live payload").toContain(
+            "total",
+          );
+          expect(warm.outputTypeScript, "the shape no longer collapses").not.toContain(
+            "data: unknown;",
+          );
+          expect(warm.note, "provenance is explicit").toContain("observed from 1 live response");
+        }),
+        Effect.gen(function* () {
+          yield* client.connections
+            .remove({
+              params: {
+                owner: "org",
+                integration: slug,
+                name: ConnectionName.make("main"),
+              },
+            })
+            .pipe(Effect.ignore);
+          yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
+        }),
+      );
+    }),
+  ),
+);

packages/core/execution/src/skills.ts

@@ -58,6 +58,7 @@ const EXECUTE_SKILL_BODY = [
   "- The `tools` object is a lazy proxy — enumerating it (`Object.keys(tools)`, spread, `for...in`) throws. Use `tools.search()` or `tools.executor.coreTools.connections.list({})` instead.",
   '- Pass an object to system tools, e.g. `tools.search({ query: "..." })`, `tools.executor.coreTools.connections.list({})`, and `tools.describe.tool({ path })`.',
   '- `tools.describe.tool()` returns compact TypeScript shapes. Use `inputTypeScript`, `outputTypeScript`, and `typeScriptDefinitions`. If the path doesn\'t resolve, the result carries `error: { code: "tool_not_found", suggestions }` — use a suggestion instead of retrying the same path.',
+  "- When `outputTypeScriptNote` is present, the `data` type was observed from live responses rather than declared by the provider: the listed fields are reliable, but the shape may be incomplete — prefer optional access for anything not listed.",
   "- For tools that return large collections (e.g. `getStates`, `getAll`), filter results in code rather than calling per-item tools.",
   "- Do not use `fetch` — all API calls go through `tools.*`.",
   "- If execution pauses for interaction, resume it with the returned `resumePayload`.",

packages/core/execution/src/tool-invoker.test.ts

@@ -877,6 +877,33 @@ describe("tool discovery", () => {
     }),
   );
 
+  it.effect("serves an observed shape with a provenance note once a schemaless tool runs", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+      const invoker = makeExecutorToolInvoker(executor, {
+        invokeOptions: { onElicitation: acceptAll },
+      });
+
+      // Cold: no declared output schema — data renders as unknown, no note.
+      const cold = yield* describeTool(executor, "github.org.main.listRepositoryIssues");
+      expect(cold.outputTypeScript).toContain("data: unknown;");
+      expect(cold.outputTypeScriptNote).toBeUndefined();
+
+      yield* invoker.invoke({
+        path: "github.org.main.listRepositoryIssues",
+        args: { owner: "executor", repo: "executor" },
+      });
+
+      // Warm: the live `[]` payload becomes the served type, marked observed
+      // both inline and via the note.
+      const warm = yield* describeTool(executor, "github.org.main.listRepositoryIssues");
+      expect(warm.outputTypeScript).toBe(
+        "{ ok: true; data: unknown[] /* observed; may be incomplete */; http?: ToolHttpMeta } | { ok: false; error: ToolError }",
+      );
+      expect(warm.outputTypeScriptNote).toContain("observed from 1 live response");
+    }),
+  );
+
   it.effect("describes a return type that accepts the sandbox invocation result", () =>
     Effect.gen(function* () {
       const executor = yield* makeSearchExecutor();

packages/core/execution/src/tool-invoker.ts

@@ -31,8 +31,12 @@ const TOOL_HTTP_META_TYPESCRIPT = "{ status: number; headers: { [k: string]: str
 const TOOL_FILE_TYPESCRIPT =
   '{ _tag: "ToolFile"; name?: string; mimeType: string; encoding: "base64"; data: string; byteLength: number; }';
 
-const wrapOutputTypeScript = (outputTypeScript?: string): string =>
-  `{ ok: true; data: ${outputTypeScript ?? "unknown"}; http?: ToolHttpMeta } | { ok: false; error: ToolError }`;
+const wrapOutputTypeScript = (outputTypeScript?: string, marker?: string): string =>
+  `{ ok: true; data: ${outputTypeScript ?? "unknown"}${marker ?? ""}; http?: ToolHttpMeta } | { ok: false; error: ToolError }`;
+
+/** Inline provenance for observed types — a model that copies only the type
+ *  string still sees the hint, since the compact render drops descriptions. */
+const OBSERVED_TYPE_MARKER = " /* observed; may be incomplete */";
 
 const withToolResultDefinitions = (
   definitions?: Record<string, string>,
@@ -76,6 +80,7 @@ type DescribedTool = {
   readonly description?: string;
   readonly inputTypeScript?: string;
   readonly outputTypeScript?: string;
+  readonly outputTypeScriptNote?: string;
   readonly typeScriptDefinitions?: Record<string, string>;
   /** Set when the path resolves to no tool — mirrors invoke's tool_not_found. */
   readonly error?: {
@@ -865,7 +870,18 @@ export const describeTool = Effect.fn("executor.tools.describe")(function* (
     name: schema.name ?? path,
     description: schema.description,
     inputTypeScript: schema.inputTypeScript,
-    outputTypeScript: wrapOutputTypeScript(schema.outputTypeScript),
+    outputTypeScript: wrapOutputTypeScript(
+      schema.outputTypeScript,
+      schema.outputSchemaSource === "observed" ? OBSERVED_TYPE_MARKER : undefined,
+    ),
+    // The compact TS render drops the schema's provenance description, so an
+    // observed (runtime-inferred) shape gets an explicit note: the model
+    // should treat the fields as reliable but not exhaustive.
+    ...(schema.outputSchemaSource === "observed"
+      ? {
+          outputTypeScriptNote: `data type observed from ${schema.outputSchemaObservations ?? 1} live response(s), not declared by the provider; fields may be incomplete.`,
+        }
+      : {}),
     typeScriptDefinitions: withToolResultDefinitions(schema.typeScriptDefinitions),
   };
   return described;

packages/core/sdk/src/executor.test.ts

@@ -673,3 +673,60 @@ describe("createExecutor", () => {
     }),
   );
 });
+
+describe("muscle memory (observed output shapes)", () => {
+  const provisioned = Effect.fn(function* () {
+    const executor = yield* makeTestExecutor({
+      plugins: [demoPlugin] as const,
+      coreTools: { webBaseUrl: "http://localhost:3000" },
+    });
+    yield* executor.demo.seed();
+    yield* executor.execute(ToolAddress.make("executor.coreTools.connections.create"), {
+      owner: "org",
+      name: String(CONN),
+      integration: String(INTEG),
+      template: String(TEMPLATE),
+      identityLabel: "Demo",
+      from: { provider: "memory", id: "secret-token" },
+    });
+    return executor;
+  });
+
+  it.effect("serves an observed output shape once a schemaless tool has run", () =>
+    Effect.gen(function* () {
+      const executor = yield* provisioned();
+
+      // Cold: `run` declares no output schema, nothing observed yet.
+      const cold = yield* executor.tools.schema(addr("run"));
+      expect(cold?.outputSchema).toBeUndefined();
+      expect(cold?.outputTypeScript).toBeUndefined();
+
+      yield* executor.execute(addr("run"), {});
+
+      // Warm: the live payload `{ ran: "run" }` becomes the served shape,
+      // with provenance marked on the schema.
+      const warm = yield* executor.tools.schema(addr("run"));
+      expect(warm?.outputSchema).toMatchObject({
+        type: "object",
+        properties: { ran: { type: "string" } },
+        required: ["ran"],
+        description: "Observed from 1 live response; fields may be incomplete.",
+      });
+      expect(warm?.outputTypeScript).toContain("ran");
+      expect(warm?.outputTypeScript).not.toBe("unknown");
+    }),
+  );
+
+  it.effect("never overrides a declared output schema with observations", () =>
+    Effect.gen(function* () {
+      const executor = yield* provisioned();
+
+      // `inspect` declares `outputSchema: { $ref: "#/$defs/Owner" }`; running
+      // it observes `{ ran: "inspect" }`, which must not displace the
+      // declared schema.
+      yield* executor.execute(addr("inspect"), { pet: { lives: 9 } });
+      const schema = yield* executor.tools.schema(addr("inspect"));
+      expect(schema?.outputSchema).toEqual({ $ref: "#/$defs/Owner" });
+    }),
+  );
+});