jhonny-editor/code/tests/hybrid-benchmark-evidence.test.ts

import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { describe, expect, it } from "vitest";
import {
  REQUIRED_TARGETS,
  verifyHybridBenchmarkEvidence,
  verifyHybridBenchmarkEvidenceWithLocalCcx,
} from "../scripts/verify-uxp-hybrid-benchmark.mjs";
import { NATIVE_SDK_HEADER_INVENTORY_SEMANTICS } from "../scripts/native-sdk-header-inventory-contract.mjs";
import { canonicalNativeSdkHeaderInventorySha256 } from "../scripts/verify-native-sdk-header-inventory.mjs";
import {
  UXP_HYBRID_ADDON_RECEIPT_SEMANTICS,
} from "../scripts/uxp-hybrid-addon-receipt-contract.mjs";
import {
  canonicalUxpHybridAddonReceiptSha256,
} from "../scripts/verify-uxp-hybrid-addon-receipt.mjs";
import {
  UXP_HYBRID_CCX_RECEIPT_LEGACY_SEMANTICS,
} from "../scripts/uxp-hybrid-ccx-receipt-contract.mjs";
import {
  canonicalUxpHybridCcxReceiptSha256,
} from "../scripts/uxp-hybrid-ccx-receipt-core.mjs";

const SHA = "a".repeat(40);
const ADDON_SHA = "b".repeat(64);
const ADDON_SHA_BY_TARGET = Object.freeze({
  "mac-x64": "b".repeat(64),
  "mac-arm64": "c".repeat(64),
  "win-x64": "d".repeat(64),
});

function sdkHeaderReceipt(sdkVersion = "UXP Hybrid SDK test fixture") {
  return {
    schemaVersion: 1,
    source: {
      sdk: "uxp-hybrid",
      sdkVersion,
      authorityUrl: "https://developer.adobe.com/premiere-pro/uxp/plugins/hybrid-plugins/",
      archiveSha256: "c".repeat(64),
      inventoryScope: "header_files_only",
      includeDirectories: ["src/api", "src/utilities"],
    },
    semantics: NATIVE_SDK_HEADER_INVENTORY_SEMANTICS,
    stats: { headers: 3, bytes: 24 },
    headers: [
      { path: "src/api/UxpAddonShared.h", bytes: 7, sha256: "d".repeat(64) },
      { path: "src/api/UxpAddonTypes.h", bytes: 8, sha256: "e".repeat(64) },
      { path: "src/utilities/UxpAddon.h", bytes: 9, sha256: "f".repeat(64) },
    ],
  };
}

function prsdkHeaderReceipt() {
  return {
    schemaVersion: 1,
    source: {
      sdk: "premiere-prsdk",
      sdkVersion: "UXP Hybrid SDK test fixture",
      authorityUrl: "https://developer.adobe.com/premiere-pro/",
      archiveSha256: "c".repeat(64),
      inventoryScope: "header_files_only",
      includeDirectories: ["Headers"],
    },
    semantics: NATIVE_SDK_HEADER_INVENTORY_SEMANTICS,
    stats: { headers: 1, bytes: 7 },
    headers: [
      { path: "Headers/PrSDKFixture.h", bytes: 7, sha256: "d".repeat(64) },
    ],
  };
}

function evidence(headerReceipt = sdkHeaderReceipt()) {
  return {
    schemaVersion: 2,
    workloadId: "weighted-energy-v1",
    configuration: { sampleCount: 30, warmupCount: 3, iterations: 4, inputLength: 131072, seed: 1337 },
    memoryMeasurement: "fixture process monitor",
    sdkHeaderReceiptSha256: canonicalNativeSdkHeaderInventorySha256(headerReceipt),
    runs: REQUIRED_TARGETS.map((target) => {
      const [platform, arch] = target.split("-");
      return {
        platform,
        arch,
        hostVersion: "26.3.0",
        sdkVersion: "UXP Hybrid SDK test fixture",
        buildMode: "Release",
        addonLoaded: true,
        addonSha256: ADDON_SHA,
        sourceCommit: SHA,
        checksumMatch: true,
        codeSigned: platform === "mac",
        notarized: platform === "mac",
        javascript: { sampleCount: 30, p50Ms: 100, p95Ms: 140, peakWorkingSetBytes: 1000 },
        native: { sampleCount: 30, p50Ms: 60, p95Ms: 80, peakWorkingSetBytes: 1050 },
      };
    }),
  };
}

function legacyEvidence() {
  const { sdkHeaderReceiptSha256, ...v1 } = evidence();
  return { ...v1, schemaVersion: 1 };
}

function addonReceipt(headerReceipt = sdkHeaderReceipt()) {
  return {
    schemaVersion: 2,
    source: {
      sdk: "uxp-hybrid",
      sdkVersion: headerReceipt.source.sdkVersion,
      sdkHeaderReceiptSha256: canonicalNativeSdkHeaderInventorySha256(headerReceipt),
      authorityUrl: "https://developer.adobe.com/premiere-pro/uxp/plugins/hybrid-plugins/build",
    },
    manifest: {
      manifestVersion: 6,
      hostApp: "premierepro",
      hostMinVersion: "26.2.0",
      addonName: "fixture-addon.uxpaddon",
      enableAddon: true,
    },
    entrypoint: { path: "main.js", bytes: 3, sha256: "e".repeat(64) },
    semantics: UXP_HYBRID_ADDON_RECEIPT_SEMANTICS,
    stats: { artifacts: 3, addonBytes: 12, entrypoints: 1, entrypointBytes: 3 },
    artifacts: [
      { target: "mac-x64", path: "mac/x64/fixture-addon.uxpaddon", bytes: 3, sha256: ADDON_SHA_BY_TARGET["mac-x64"] },
      { target: "mac-arm64", path: "mac/arm64/fixture-addon.uxpaddon", bytes: 4, sha256: ADDON_SHA_BY_TARGET["mac-arm64"] },
      { target: "win-x64", path: "win/x64/fixture-addon.uxpaddon", bytes: 5, sha256: ADDON_SHA_BY_TARGET["win-x64"] },
    ],
  };
}

function ccxReceipt(headerReceipt = sdkHeaderReceipt(), addon = addonReceipt(headerReceipt)) {
  return {
    schemaVersion: 1,
    source: {
      sdk: "uxp-hybrid",
      sdkVersion: headerReceipt.source.sdkVersion,
      sdkHeaderReceiptSha256: canonicalNativeSdkHeaderInventorySha256(headerReceipt),
      addonReceiptSha256: canonicalUxpHybridAddonReceiptSha256(addon),
      authorityUrl: "https://developer.adobe.com/premiere-pro/uxp/plugins/distribution/package/",
    },
    archive: { format: "zip", bytes: 64, sha256: "f".repeat(64) },
    manifest: {
      bytes: 32,
      sha256: "1".repeat(64),
      idSha256: "2".repeat(64),
      idLength: 7,
      manifestVersion: 6,
      hostApp: "premierepro",
      hostMinVersion: "26.2.0",
      addonName: "fixture-addon.uxpaddon",
      enableAddon: true,
    },
    semantics: UXP_HYBRID_CCX_RECEIPT_LEGACY_SEMANTICS,
    stats: { artifacts: 3, addonBytes: 12, entrypoints: 1, entrypointBytes: 3 },
  };
}

function packageEvidence(headerReceipt = sdkHeaderReceipt(), addon = addonReceipt(headerReceipt), ccx = ccxReceipt(headerReceipt, addon)) {
  return {
    ...evidence(headerReceipt),
    schemaVersion: 3,
    addonReceiptSha256: canonicalUxpHybridAddonReceiptSha256(addon),
    ccxReceiptSha256: canonicalUxpHybridCcxReceiptSha256(ccx),
    runs: REQUIRED_TARGETS.map((target) => {
      const [platform, arch] = target.split("-");
      return {
        platform,
        arch,
        hostVersion: "26.3.0",
        sdkVersion: headerReceipt.source.sdkVersion,
        buildMode: "Release",
        addonLoaded: true,
        addonSha256: ADDON_SHA_BY_TARGET[target],
        sourceCommit: SHA,
        checksumMatch: true,
        codeSigned: platform === "mac",
        notarized: platform === "mac",
        javascript: { sampleCount: 30, p50Ms: 100, p95Ms: 140, peakWorkingSetBytes: 1000 },
        native: { sampleCount: 30, p50Ms: 60, p95Ms: 80, peakWorkingSetBytes: 1050 },
      };
    }),
  };
}

describe("UXP hybrid benchmark evidence verifier", () => {
  it("keeps frozen v1/v2 schemas and verifier paths available while making package binding v3", () => {
    const v1Schema = JSON.parse(readFileSync("benchmarks/uxp-hybrid/evidence.v1.schema.json", "utf8"));
    const v2Schema = JSON.parse(readFileSync("benchmarks/uxp-hybrid/evidence.v2.schema.json", "utf8"));
    const v3Schema = JSON.parse(readFileSync("benchmarks/uxp-hybrid/evidence.schema.json", "utf8"));
    expect(v1Schema.$id).toBe("https://premiere-pro-mcp.com/schemas/uxp-hybrid-benchmark-evidence-v1.json");
    expect(v1Schema.properties.schemaVersion.const).toBe(1);
    expect(v1Schema.required).not.toContain("sdkHeaderReceiptSha256");
    expect(v2Schema.$id).toBe("https://premiere-pro-mcp.com/schemas/uxp-hybrid-benchmark-evidence-v2.json");
    expect(v2Schema.properties.schemaVersion.const).toBe(2);
    expect(v2Schema.required).toContain("sdkHeaderReceiptSha256");
    expect(v3Schema.$id).toBe("https://premiere-pro-mcp.com/schemas/uxp-hybrid-benchmark-evidence-v3.json");
    expect(v3Schema.properties.schemaVersion.const).toBe(3);
    expect(v3Schema.required).toEqual(expect.arrayContaining(["sdkHeaderReceiptSha256", "addonReceiptSha256", "ccxReceiptSha256"]));
    expect(verifyHybridBenchmarkEvidence(legacyEvidence())).toEqual({
      promotionEligible: true,
      errors: [],
      targets: ["mac-arm64", "mac-x64", "win-x64"],
      thresholds: { minimumSpeedupPercent: 30, maximumMemoryRegressionPercent: 10 },
      verificationBoundary: "submitted_cross_platform_release_build_evidence",
    });
  });

  it("requires all release targets, matching output, speed, memory, signing, and one commit", () => {
    const receipt = sdkHeaderReceipt();
    expect(verifyHybridBenchmarkEvidence(evidence(receipt), { sdkHeaderReceipt: receipt })).toEqual({
      promotionEligible: true,
      errors: [],
      targets: ["mac-arm64", "mac-x64", "win-x64"],
      thresholds: { minimumSpeedupPercent: 30, maximumMemoryRegressionPercent: 10 },
      verificationBoundary: "submitted_cross_platform_release_build_and_verified_sdk_receipt_evidence",
    });
  });

  it("fails closed when a target is missing or a percentile misses the threshold", () => {
    const receipt = sdkHeaderReceipt();
    const value = evidence(receipt);
    value.runs.pop();
    value.runs[0].native.p95Ms = 110;
    const result = verifyHybridBenchmarkEvidence(value, { sdkHeaderReceipt: receipt });
    expect(result.promotionEligible).toBe(false);
    expect(result.errors).toEqual(expect.arrayContaining([
      expect.stringContaining("p95 speedup"),
      expect.stringContaining("Missing required mac-arm64 evidence"),
    ]));
  });

  it("requires a structurally verified matching Hybrid receipt and rejects undocumented fields", () => {
    const receipt = sdkHeaderReceipt();
    const value = evidence(receipt);
    expect(verifyHybridBenchmarkEvidence(value).errors).toContain("A verified UXP Hybrid SDK header receipt is required.");

    const mismatchedDigest = { ...value, sdkHeaderReceiptSha256: "0".repeat(64) };
    expect(verifyHybridBenchmarkEvidence(mismatchedDigest, { sdkHeaderReceipt: receipt }).errors)
      .toContain("sdkHeaderReceiptSha256 does not match the verified SDK header receipt.");

    const mismatchedVersionReceipt = sdkHeaderReceipt("UXP Hybrid SDK other fixture");
    expect(verifyHybridBenchmarkEvidence(value, { sdkHeaderReceipt: mismatchedVersionReceipt }).errors)
      .toEqual(expect.arrayContaining([
        "SDK header receipt sdkVersion must match all benchmark runs.",
        "sdkHeaderReceiptSha256 does not match the verified SDK header receipt.",
      ]));

    const wrongSdkReceipt = prsdkHeaderReceipt();
    expect(verifyHybridBenchmarkEvidence(value, { sdkHeaderReceipt: wrongSdkReceipt }).errors)
      .toEqual(expect.arrayContaining([
        "SDK header receipt must identify uxp-hybrid.",
        "sdkHeaderReceiptSha256 does not match the verified SDK header receipt.",
      ]));

    const withExtraField = { ...value, privateSdkPath: "C:Headers" };
    expect(verifyHybridBenchmarkEvidence(withExtraField, { sdkHeaderReceipt: receipt }).errors)
      .toContain("Evidence must contain only the documented benchmark receipt fields.");

    const withExtraRunField = {
      ...value,
      runs: [{ ...value.runs[0], localAddonPath: "C:Headers" }, ...value.runs.slice(1)],
    };
    expect(verifyHybridBenchmarkEvidence(withExtraRunField, { sdkHeaderReceipt: receipt }).errors)
      .toContain("runs[0] must contain only the documented benchmark fields.");

    const withExtraMetricField = {
      ...value,
      runs: [{ ...value.runs[0], native: { ...value.runs[0].native, rawSamples: [60] } }, ...value.runs.slice(1)],
    };
    expect(verifyHybridBenchmarkEvidence(withExtraMetricField, { sdkHeaderReceipt: receipt }).errors)
      .toContain("runs[0].native must contain only the documented metric fields.");
  });

  it("binds each target run to a verified current addon-layout and CCX receipt chain", async () => {
    const headerReceipt = sdkHeaderReceipt();
    const addon = addonReceipt(headerReceipt);
    const ccx = ccxReceipt(headerReceipt, addon);
    const value = packageEvidence(headerReceipt, addon, ccx);
    expect(verifyHybridBenchmarkEvidence(value, {
      sdkHeaderReceipt: headerReceipt,
      addonReceipt: addon,
      ccxReceipt: ccx,
    })).toEqual({
      promotionEligible: false,
      errors: ["A local CCX archive must be rechecked for schemaVersion 3."],
      targets: ["mac-arm64", "mac-x64", "win-x64"],
      thresholds: { minimumSpeedupPercent: 30, maximumMemoryRegressionPercent: 10 },
      verificationBoundary: "submitted_cross_platform_release_build_and_verified_sdk_addon_ccx_receipt_evidence",
    });

    const mismatchedArtifact = packageEvidence(headerReceipt, addon, ccx);
    mismatchedArtifact.runs[0].addonSha256 = "0".repeat(64);
    expect(verifyHybridBenchmarkEvidence(mismatchedArtifact, {
      sdkHeaderReceipt: headerReceipt,
      addonReceipt: addon,
      ccxReceipt: ccx,
    }).errors).toContain("runs for win-x64 must match the corresponding addon-layout artifact SHA-256.");

    const withoutArchive = await verifyHybridBenchmarkEvidenceWithLocalCcx(value, {
      sdkHeaderReceipt: headerReceipt,
      addonReceipt: addon,
      ccxReceipt: ccx,
    });
    expect(withoutArchive).toMatchObject({
      promotionEligible: false,
      errors: ["A local CCX archive path is required for schemaVersion 3."],
    });
  });

  it("requires a local receipt path for v2 without printing it while retaining v1 CLI compatibility", () => {
    const directory = mkdtempSync(join(tmpdir(), "premiere-hybrid-benchmark-"));
    const input = join(directory, "evidence.json");
    const receiptPath = join(directory, "receipt.json");
    const receipt = sdkHeaderReceipt();
    try {
      writeFileSync(input, `${JSON.stringify(evidence(receipt), null, 2)}\n`);
      writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`);
      const missingReceipt = spawnSync(process.execPath, ["scripts/verify-uxp-hybrid-benchmark.mjs", "--input", input], { encoding: "utf8" });
      expect(missingReceipt.status).not.toBe(0);
      expect(missingReceipt.stdout).toContain("A verified UXP Hybrid SDK header receipt is required.");

      writeFileSync(input, `${JSON.stringify(legacyEvidence(), null, 2)}\n`);
      const legacy = spawnSync(process.execPath, ["scripts/verify-uxp-hybrid-benchmark.mjs", "--input", input], { encoding: "utf8" });
      expect(legacy.status).toBe(0);
      expect(legacy.stdout).toContain('"verificationBoundary": "submitted_cross_platform_release_build_evidence"');

      writeFileSync(input, `${JSON.stringify(evidence(receipt), null, 2)}\n`);

      const verified = spawnSync(process.execPath, ["scripts/verify-uxp-hybrid-benchmark.mjs", "--input", input, "--sdk-header-receipt", receiptPath], { encoding: "utf8" });
      expect(verified.status).toBe(0);
      expect(verified.stdout).toContain('"promotionEligible": true');
      expect(verified.stdout).not.toContain("UxpAddonShared.h");

      const unreadableReceipt = join(directory, "do-not-disclose-this-receipt.json");
      const unreadable = spawnSync(process.execPath, ["scripts/verify-uxp-hybrid-benchmark.mjs", "--input", input, "--sdk-header-receipt", unreadableReceipt], { encoding: "utf8" });
      expect(unreadable.status).not.toBe(0);
      expect(unreadable.stderr).toContain("SDK header receipt must be a readable JSON document.");
      expect(unreadable.stderr).not.toContain(unreadableReceipt);
    } finally {
      rmSync(directory, { recursive: true, force: true });
    }
  });
});